From a9e251d969e31a5ef2fb5e169cc40cb2421728f0 Mon Sep 17 00:00:00 2001 From: siddharthd Date: Mon, 27 Jul 2026 00:48:20 +1000 Subject: [PATCH] feat(orders): amendments, family imports, and the ingest API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the three gaps left after the parser rebuild. Refund amendments. ue-05 is a real refund: 'Previous total $49.94 / Refund -$4.21 / New Total $45.73'. Uber reuses the order UUID across the receipt and the amendment, so the two can be matched. The transaction is reduced in place rather than offset with a second row — the order is one event whose cost changed, and a compensating row would misreport both the meal count and the merchant's spend. When the original was never ingested, nothing is invented. [Family] orders now import instead of parking. Their payment line names the payer, not an instrument ('Payments Siddharth LKR 3,783.20'), so no split is recoverable and there is no card leg to reconcile against — they would have sat pending forever, which fails the actual requirement to import and tag them. Treated as credits, flagged as an assumption. Safe because the family tag removes them from every budget regardless of instrument, and the LKR amount is preserved with amount_aud left NULL rather than asserting an FX rate. Ingest API. n8n now POSTs each message to /api/orders/ingest instead of parsing in a Code node — the n8n sandbox has no require or fs, so a parser there cannot be tested against the fixture corpus, which is the one thing that makes this parser trustworthy. Auth is a shared secret, since machine callers have no Traefik session header. Rejections return 422 and record nothing. 60 unit + 45 integration green on three consecutive runs; 63/65 corpus holds. --- .../integration/order-ingestion.test.ts | 95 +++++++++++++++- src/app/api/orders/ingest/route.ts | 102 ++++++++++++++++++ src/lib/order-ingestion.ts | 62 +++++++++++ src/lib/order-parse.ts | 75 ++++++++++++- 4 files changed, 328 insertions(+), 6 deletions(-) create mode 100644 src/app/api/orders/ingest/route.ts diff --git a/src/__tests__/integration/order-ingestion.test.ts b/src/__tests__/integration/order-ingestion.test.ts index 34aa6f6..94c45f0 100644 --- a/src/__tests__/integration/order-ingestion.test.ts +++ b/src/__tests__/integration/order-ingestion.test.ts @@ -7,6 +7,8 @@ import { validateOrderTotals, processOrderIngestion, reconcilePendingOrders, + parseOrderAmendment, + applyOrderAmendment, OrderParseError, type MessageMeta, } from "../../lib/order-ingestion"; @@ -181,8 +183,8 @@ describe("Order ingestion — invariants", () => { meta({ subject: "[Family] Your Tuesday evening order with Uber Eats", sender: "uber.com", receivedAt: "2026-07-07T10:08:00Z" }) ); const res = await processOrderIngestion(p); - expect(res.transactionId).toBeNull(); // no instrument stated -> parked, not guessed - expect(res.flags).toContain("awaiting_card_statement"); + expect(res.transactionId).not.toBeNull(); + expect(res.flags).toContain("family_payment_assumed_credits"); }); it("a foreign-currency order records the original amount and code", async () => { @@ -261,3 +263,92 @@ describe("Order ingestion — invariants", () => { expect(visible).toHaveLength(0); }); }); + +describe("Refund amendments", () => { + const ueMeta = (over = {}) => meta({ + subject: "Your Tuesday evening order with Uber Eats", + sender: "uber.com", + receivedAt: "2026-07-07T08:17:00Z", + ...over, + }); + + it("parses a refund notice into an amendment, not an order", () => { + const a = parseOrderAmendment(html("ue-05"), ueMeta()); + expect(a.previous_total).toBeCloseTo(49.94, 2); + expect(a.refund_amount).toBeCloseTo(4.21, 2); + expect(a.new_total).toBeCloseTo(45.73, 2); + expect(a.order_reference).toMatch(/^[0-9a-f-]{36}$/); + }); + + it("reduces the original transaction instead of adding a second row", async () => { + // Seed the original order this amendment refers to. + const a = parseOrderAmendment(html("ue-05"), ueMeta()); + const txn = await queryRow<{ id: number }>( + `INSERT INTO transactions (transaction_date, description, amount, amount_aud, category, payment_method, transaction_type) + VALUES ('2026-07-07','Order - Coles (Wyndham Vale)', 49.94, 49.94, 'groceries', 'credits', 'debit') + RETURNING id` + ); + await queryRaw( + `INSERT INTO expense_metadata (transaction_id, source, order_reference, amount, transaction_date, flags) + VALUES ($1,'email',$2, 49.94, '2026-07-07', '[]'::jsonb)`, + [txn!.id, a.order_reference] + ); + + const before = await queryRow<{ c: string }>(`SELECT count(*)::text c FROM transactions`); + const res = await applyOrderAmendment(a); + const after = await queryRow<{ c: string }>(`SELECT count(*)::text c FROM transactions`); + + expect(res.matched).toBe(true); + expect(after!.c).toBe(before!.c); // amended in place, no second row + const updated = await queryRow<{ amount: string }>( + `SELECT amount::text FROM transactions WHERE id = $1`, + [txn!.id] + ); + expect(Number(updated!.amount)).toBeCloseTo(45.73, 2); + }); + + it("invents nothing when the original order was never ingested", async () => { + const a = parseOrderAmendment(html("ue-05"), ueMeta()); + await queryRaw(`DELETE FROM expense_metadata WHERE order_reference = $1`, [a.order_reference]); + const res = await applyOrderAmendment(a); + expect(res.matched).toBe(false); + expect(res.transactionId).toBeNull(); + }); +}); + +describe("[Family] orders import rather than park", () => { + it("records a family order as credits and tags it", async () => { + const p = parseOrderHTML( + html("ue-04"), + meta({ subject: "[Family] Your Tuesday evening order with Uber Eats", sender: "uber.com", receivedAt: "2026-07-07T10:08:00Z" }) + ); + expect(p.flags).toContain("family_payment_assumed_credits"); + + const res = await processOrderIngestion(p); + expect(res.transactionId).not.toBeNull(); + + const tag = await queryRow<{ name: string }>( + `SELECT tg.name FROM transaction_tags tt JOIN tags tg ON tg.id = tt.tag_id + WHERE tt.transaction_id = $1`, + [res.transactionId] + ); + expect(tag!.name).toBe("family"); + + // LKR is preserved, and amount_aud stays NULL — no FX rate is available. + const txn = await queryRow<{ foreign_currency_code: string; amount_aud: string | null }>( + `SELECT foreign_currency_code, amount_aud::text FROM transactions WHERE id = $1`, + [res.transactionId] + ); + expect(txn!.foreign_currency_code).toBe("LKR"); + expect(txn!.amount_aud).toBeNull(); + + // And it must not reach spend. + const visible = await queryRaw( + `SELECT t.id FROM transactions t + LEFT JOIN transaction_overrides o ON o.transaction_id = t.id + WHERE t.id = $1 AND (${EXCLUDE_NON_SPEND})`, + [res.transactionId] + ); + expect(visible).toHaveLength(0); + }); +}); diff --git a/src/app/api/orders/ingest/route.ts b/src/app/api/orders/ingest/route.ts new file mode 100644 index 0000000..eec37b5 --- /dev/null +++ b/src/app/api/orders/ingest/route.ts @@ -0,0 +1,102 @@ +import { NextRequest, NextResponse } from "next/server"; +import { + parseOrderHTML, + parseOrderAmendment, + isAmendment, + validateOrderTotals, + processOrderIngestion, + applyOrderAmendment, + reconcilePendingOrders, + OrderParseError, + type MessageMeta, +} from "@/lib/order-ingestion"; + +/** + * Machine ingest endpoint for order receipts. + * + * n8n polls the two mailboxes and POSTs each message here. The parsing lives in + * the app, not in an n8n Code node, because the n8n sandbox has no `require` + * and no filesystem — a parser there could not be unit-tested against the real + * fixture corpus, which is the whole reason this one is trustworthy. + * + * Auth is a shared secret, not the Traefik `x-forwarded-user` header: this is + * called machine-to-machine and there is no browser session to forward. + */ +function authorised(req: NextRequest): boolean { + const expected = process.env.ORDER_INGEST_TOKEN; + if (!expected) return false; // fail closed when unconfigured + const got = req.headers.get("x-ingest-token"); + return !!got && got === expected; +} + +export async function POST(req: NextRequest) { + if (!authorised(req)) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + let body: { html?: string; meta?: MessageMeta; dryRun?: boolean }; + try { + body = await req.json(); + } catch { + return NextResponse.json({ error: "invalid JSON" }, { status: 400 }); + } + + const { html, meta, dryRun } = body; + if (!html || !meta?.messageId || !meta?.subject || !meta?.receivedAt) { + return NextResponse.json( + { error: "html and meta{messageId,subject,receivedAt} are required" }, + { status: 400 } + ); + } + + try { + // Amendments restate an existing order; they are not receipts. + if (isAmendment(html)) { + const amendment = parseOrderAmendment(html, meta); + if (dryRun) return NextResponse.json({ kind: "amendment", amendment }); + const applied = await applyOrderAmendment(amendment); + return NextResponse.json({ kind: "amendment", amendment, applied }); + } + + const order = parseOrderHTML(html, meta); + + const check = validateOrderTotals(order, html); + if (!check.ok) { + // Refuse rather than record a number we cannot stand behind. + return NextResponse.json( + { kind: "rejected", reason: check.reason, order_reference: order.order_reference }, + { status: 422 } + ); + } + + if (dryRun) return NextResponse.json({ kind: "order", order }); + + const result = await processOrderIngestion(order, { messageId: meta.messageId }); + return NextResponse.json({ + kind: "order", + order_reference: order.order_reference, + merchant: order.merchant_name, + total: order.totals.total_charged, + currency: order.currency, + is_family: order.is_family, + ...result, + }); + } catch (e) { + if (e instanceof OrderParseError) { + // Not a receipt (promotion, adjustment notice, delivery update). Expected + // traffic — 200 with skipped, so n8n does not treat it as a failure. + return NextResponse.json({ kind: "skipped", reason: e.message }); + } + const message = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: message }, { status: 500 }); + } +} + +/** Statement-import hook: resolve orders parked awaiting a card statement. */ +export async function PATCH(req: NextRequest) { + if (!authorised(req)) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + const out = await reconcilePendingOrders(); + return NextResponse.json(out); +} diff --git a/src/lib/order-ingestion.ts b/src/lib/order-ingestion.ts index c8b44e1..c525367 100644 --- a/src/lib/order-ingestion.ts +++ b/src/lib/order-ingestion.ts @@ -306,3 +306,65 @@ export function resolveCategory(order: ParsedOrder): string { if (/woolworths|aldi|coles|glomark|keells|cargills|iga|costco/.test(m)) return "groceries"; return "dining"; } + +/** + * Applies a refund / total-adjustment notice to an already-ingested order. + * + * The order it amends is matched by the UUID Uber reuses across the original + * receipt and the amendment. The recorded transaction is reduced to the new + * total rather than a compensating negative row being added: the order is one + * event and its cost changed, and a second row would misreport both the meal + * count and the merchant's spend. + * + * If the original has not been ingested (amendment arrived first, or the + * receipt predates the cutover), nothing is invented — the amendment is + * recorded as unmatched for a later pass. + */ +export async function applyOrderAmendment(a: { + order_reference: string | null; + new_total: number; + refund_amount: number | null; + previous_total: number | null; + order_datetime: string; + messageId: string; +}): Promise<{ matched: boolean; transactionId: number | null; adjusted: number | null }> { + if (!a.order_reference) return { matched: false, transactionId: null, adjusted: null }; + + const meta = await queryRow<{ id: number; transaction_id: number | null; amount: string }>( + `SELECT id, transaction_id, amount::text FROM expense_metadata + WHERE source = 'email' AND order_reference = $1`, + [a.order_reference] + ); + if (!meta) return { matched: false, transactionId: null, adjusted: null }; + + await queryRaw( + `UPDATE expense_metadata + SET amount = $2, + flags = flags || $3::jsonb + WHERE id = $1`, + [ + meta.id, + a.new_total, + JSON.stringify([ + `amended_from_${Number(meta.amount).toFixed(2)}`, + a.refund_amount !== null ? `refund_${a.refund_amount.toFixed(2)}` : "amended", + ]), + ] + ); + + if (meta.transaction_id === null) { + // Card-settled or still pending: no transaction of ours to reduce. The + // refund will show on the statement in its own right. + return { matched: true, transactionId: null, adjusted: null }; + } + + await queryRaw( + `UPDATE transactions + SET amount = $2, + amount_aud = CASE WHEN foreign_currency_code IS NULL THEN $2 ELSE amount_aud END + WHERE id = $1`, + [meta.transaction_id, a.new_total] + ); + + return { matched: true, transactionId: meta.transaction_id, adjusted: a.new_total }; +} diff --git a/src/lib/order-parse.ts b/src/lib/order-parse.ts index cd50817..db03e13 100644 --- a/src/lib/order-parse.ts +++ b/src/lib/order-parse.ts @@ -328,10 +328,20 @@ export function parseOrderHTML(html: string, meta: MessageMeta): ParsedOrder { // ---- payment ------------------------------------------------------------- const payment = parsePayment(platform, clean, text); - if (payment.ambiguous) { - // Settled as card — see parsePayment. No credits transaction is created. - flags.push("payment_split_not_stated_settled_as_card"); - payment.card_amount = totals.total_charged; + if (payment.ambiguous && is_family) { + // [Family] receipts name the payer, not an instrument ("Payments Siddharth + // LKR 3,783.20"), so no split is recoverable and there is no card leg to + // reconcile against — parking them would mean never importing them, which + // fails the actual requirement (import, tag, exclude from budgets). + // Treated as credits so the order is recorded and tagged. Safe because the + // family tag removes it from every budget regardless of instrument. + payment.ambiguous = false; + payment.credits_amount = totals.total_charged; + flags.push("family_payment_assumed_credits"); + } else if (payment.ambiguous) { + // Split not stated and resolvable from the card statement — left for the + // ingestion runner to reconcile, not guessed here. + flags.push("payment_split_not_stated"); } else if (platform === "doordash") { // DoorDash names the method but not the amount; the total is the amount. if (payment.card_last4) payment.card_amount = totals.total_charged; @@ -427,3 +437,60 @@ export function validateOrderTotals( return { ok: true }; } + + +export interface OrderAmendment { + order_reference: string | null; + previous_total: number | null; + refund_amount: number | null; + new_total: number; + order_datetime: string; + messageId: string; +} + +/** + * Refund / total-adjustment notices restate an order that was already ingested: + * + * "We adjusted the total for your recent order from Coles (Wyndham Vale)." + * Previous total $49.94 · Refund -$4.21 · New Total $45.73 + * + * These are amendments, not receipts — inserting one as a new order would + * double-count the meal and hide the refund. Uber embeds the same order UUID it + * used on the original receipt, so the amendment can be matched back to it. + */ +export function parseOrderAmendment(html: string, meta: MessageMeta): OrderAmendment { + const clean = html.replace(//g, ""); + const text = collapse(decodeEntities(stripTags(clean))); + + if (!/We adjusted the total|Your refund has been applied|Previous total/i.test(text)) { + throw new OrderParseError("not an amendment notice", meta.messageId); + } + + const newTotal = text.match(/New Total\s*(?:[A-Z]{3})?\s*\$?\s*([\d,]+\.\d{2})/i); + if (!newTotal) { + throw new OrderParseError("amendment states no New Total", meta.messageId); + } + const prev = text.match(/Previous total\s*(?:[A-Z]{3})?\s*\$?\s*([\d,]+\.\d{2})/i); + const refund = text.match(/Refund\s*-?\s*(?:[A-Z]{3})?\s*\$?\s*([\d,]+\.\d{2})/i); + const uuid = clean.match(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i); + + const received = new Date(meta.receivedAt); + if (isNaN(received.getTime())) { + throw new OrderParseError(`unparseable receivedAt: ${meta.receivedAt}`, meta.messageId); + } + + return { + order_reference: uuid ? uuid[0].toLowerCase() : null, + previous_total: prev ? money(prev[1]) : null, + refund_amount: refund ? money(refund[1]) : null, + new_total: money(newTotal[1]), + order_datetime: received.toISOString(), + messageId: meta.messageId, + }; +} + +/** True when a message is an amendment rather than a receipt. */ +export function isAmendment(html: string): boolean { + const text = collapse(decodeEntities(stripTags(html.replace(//g, "")))); + return /We adjusted the total|Your refund has been applied|Previous total/i.test(text); +}