diff --git a/prisma/migrations/0020_order_match_provenance/migration.sql b/prisma/migrations/0020_order_match_provenance/migration.sql new file mode 100644 index 0000000..5fd9dab --- /dev/null +++ b/prisma/migrations/0020_order_match_provenance/migration.sql @@ -0,0 +1,13 @@ +-- Which statement line settled an order's card leg. +-- +-- Without this, reconcileCardLeg has no way to know a charge has already been +-- consumed, so two orders on the same card inside the match window both bind to +-- it and each books its own credits remainder -- double-counting spend. +ALTER TABLE expense_metadata + ADD COLUMN IF NOT EXISTS matched_transaction_id INTEGER + REFERENCES transactions(id) ON DELETE SET NULL; + +-- One statement line settles at most one order. +CREATE UNIQUE INDEX IF NOT EXISTS uq_expense_matched_txn + ON expense_metadata (matched_transaction_id) + WHERE matched_transaction_id IS NOT NULL; diff --git a/src/__tests__/integration/order-ingest-api.test.ts b/src/__tests__/integration/order-ingest-api.test.ts new file mode 100644 index 0000000..81ed5bf --- /dev/null +++ b/src/__tests__/integration/order-ingest-api.test.ts @@ -0,0 +1,106 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { readFileSync } from "fs"; +import { resolve } from "path"; +import { queryRaw } from "../../lib/db"; + +/** + * The HTTP path had no tests at all — which is how three defects reached the + * branch through a suite of 105 green ones. These exercise the route handler + * directly (no server needed) so the auth gate and the error taxonomy are + * actually covered. + */ +const dir = resolve(__dirname, "../fixtures/orders/real"); +const html = (f: string) => readFileSync(resolve(dir, `${f}.html`), "utf-8"); + +const TOKEN = "test-ingest-token"; +let POST: any; + +const req = (body: unknown, token: string | null = TOKEN) => + ({ + headers: { get: (h: string) => (h === "x-ingest-token" ? token : null) }, + json: async () => body, + }) as any; + +beforeAll(async () => { + process.env.ORDER_INGEST_TOKEN = TOKEN; + ({ POST } = await import("../../app/api/orders/ingest/route")); +}); + +describe("ingest API — auth", () => { + it("rejects a missing token", async () => { + const res = await POST(req({ html: "x", meta: {} }, null)); + expect(res.status).toBe(401); + }); + + it("rejects a wrong token", async () => { + const res = await POST(req({ html: "x", meta: {} }, "nope")); + expect(res.status).toBe(401); + }); + + it("rejects a malformed body", async () => { + const res = await POST(req({ html: "only html" })); + expect(res.status).toBe(400); + }); +}); + +describe("ingest API — error taxonomy", () => { + const meta = (over = {}) => ({ + messageId: `api-${Math.random().toString(36).slice(2)}`, + subject: "Order Confirmation for Siddharth from Mad Mex", + receivedAt: "2026-07-16T03:34:00Z", + sender: "DoorDash Order ", + ...over, + }); + + it("a non-receipt is 200 and silent — it must not alert", async () => { + // A newsletter: real traffic, correctly ignored. + const res = await POST(req({ html: html("dd-01"), meta: meta({ subject: "Newsletter", sender: "promo@example.com" }) })); + expect(res.status).toBe(200); + expect((await res.json()).kind).toBe("skipped"); + }); + + it("an adjustment notice is 200 and silent", async () => { + const res = await POST(req({ + html: html("dd-08"), + meta: meta({ subject: "Order Confirmation for Siddharth from ALDI" }), + })); + expect(res.status).toBe(200); + expect((await res.json()).kind).toBe("skipped"); + }); + + it("a receipt that cannot be parsed is 422 so it ALERTS", async () => { + // A DoorDash receipt with its totals stripped out — i.e. what a provider + // template change looks like. Previously this returned 200 and vanished. + const broken = html("dd-01") + .replace(/Total Charged/g, "Gesamtbetrag") + .replace(/Total:/g, "Summe:"); + const res = await POST(req({ html: broken, meta: meta() })); + expect(res.status).toBe(422); + const body = await res.json(); + expect(body.kind).toBe("parse_failed"); + expect(body.reason).toMatch(/total/i); + }); + + it("a refund is routed to the amendment path, not ingestion", async () => { + const res = await POST(req({ + html: html("ue-05"), + meta: meta({ subject: "Your Tuesday evening order with Uber Eats", sender: "uber.com" }), + dryRun: true, + })); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.kind).toBe("amendment"); + expect(body.amendment.new_total).toBeCloseTo(45.73, 2); + }); + + it("a good receipt ingests", async () => { + await queryRaw(`DELETE FROM expense_metadata WHERE source = 'email'`); + await queryRaw(`DELETE FROM transactions WHERE description LIKE 'Order - %'`); + const res = await POST(req({ html: html("dd-01"), meta: meta({ messageId: "api-ok-1" }) })); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.kind).toBe("order"); + expect(body.total).toBe(14.64); + expect(body.transactionId).not.toBeNull(); + }); +}); diff --git a/src/__tests__/integration/order-ingestion.test.ts b/src/__tests__/integration/order-ingestion.test.ts index 94c45f0..09a6c98 100644 --- a/src/__tests__/integration/order-ingestion.test.ts +++ b/src/__tests__/integration/order-ingestion.test.ts @@ -10,6 +10,7 @@ import { parseOrderAmendment, applyOrderAmendment, OrderParseError, + NotAReceiptError, type MessageMeta, } from "../../lib/order-ingestion"; import { EXCLUDE_NON_SPEND } from "../../lib/analytics-sql"; @@ -113,9 +114,12 @@ describe("Order parsing — real receipts", () => { }); it("rejects an order-adjustment notice rather than booking $0.00", () => { + // NotAReceiptError, not OrderParseError: this is expected traffic, so it + // must be skipped silently. Only a receipt that fails to parse should + // alert — see the ingest API's error taxonomy. expect(() => parseOrderHTML(html("dd-08"), meta({ subject: "Order Confirmation for Siddharth from ALDI" })) - ).toThrow(OrderParseError); + ).toThrow(NotAReceiptError); }); it("rejects a refund notice rather than inserting a duplicate order", () => { diff --git a/src/__tests__/unit/order-ingestion.test.ts b/src/__tests__/unit/order-ingestion.test.ts index 2bc1081..c51c44d 100644 --- a/src/__tests__/unit/order-ingestion.test.ts +++ b/src/__tests__/unit/order-ingestion.test.ts @@ -6,6 +6,7 @@ import { validateOrderTotals, resolveCategory, OrderParseError, + NotAReceiptError, type MessageMeta, type ParsedOrder, } from "../../lib/order-ingestion"; @@ -125,7 +126,7 @@ describe("resolveCategory", () => { describe("parse guards", () => { it("throws on a body too short to be a receipt", () => { - expect(() => parseOrderHTML("", meta())).toThrow(OrderParseError); + expect(() => parseOrderHTML("", meta())).toThrow(NotAReceiptError); }); it("throws rather than inventing a platform", () => { @@ -134,3 +135,32 @@ describe("parse guards", () => { ).toThrow(/platform/i); }); }); + +describe("order_reference anchoring", () => { + it("takes Uber's tripReference, not the first UUID in the document", () => { + const p = parseOrderHTML( + html("ue-00"), + meta({ subject: "Your Wednesday afternoon order with Uber Eats", sender: "uber.com" }) + ); + // The UUID the PDF redirect actually resolves to for this receipt. + expect(p.order_reference).toBe("34d6b4ee-da8f-5029-8d14-bd359617c8e9"); + expect(p.flags).not.toContain("order_uuid_ambiguous"); + }); + + it("is stable across repeated parses of the same message", () => { + const m = meta({ subject: "Your Wednesday afternoon order with Uber Eats", sender: "uber.com" }); + const a = parseOrderHTML(html("ue-00"), m).order_reference; + const b = parseOrderHTML(html("ue-00"), m).order_reference; + expect(a).toBe(b); + }); + + it("flags ambiguity only when several UUIDs and no anchor", () => { + // Strip the anchor from a receipt that carries multiple UUIDs (ue-09 has 6). + const stripped = html("ue-09").replace(/tripReference/gi, "notTheAnchor"); + const p = parseOrderHTML( + stripped, + meta({ subject: "[Family] Your Sunday evening order with Uber Eats", sender: "uber.com" }) + ); + expect(p.flags).toContain("order_uuid_ambiguous"); + }); +}); diff --git a/src/app/api/orders/ingest/route.ts b/src/app/api/orders/ingest/route.ts index eec37b5..c628e06 100644 --- a/src/app/api/orders/ingest/route.ts +++ b/src/app/api/orders/ingest/route.ts @@ -8,6 +8,7 @@ import { applyOrderAmendment, reconcilePendingOrders, OrderParseError, + NotAReceiptError, type MessageMeta, } from "@/lib/order-ingestion"; @@ -82,11 +83,25 @@ export async function POST(req: NextRequest) { ...result, }); } catch (e) { - if (e instanceof OrderParseError) { - // Not a receipt (promotion, adjustment notice, delivery update). Expected - // traffic — 200 with skipped, so n8n does not treat it as a failure. + // Not a receipt: promotions, delivery updates, adjustment and refund + // notices. Expected traffic — 200 and silent, or the alert channel fills + // with noise and stops being read. + if (e instanceof NotAReceiptError) { return NextResponse.json({ kind: "skipped", reason: e.message }); } + + // IS a receipt, could not be parsed. This is the failure that matters and + // it must be loud: a provider template change breaks every order at once, + // and the only other symptom is spend quietly ceasing to appear. Returning + // 200 here — as this route originally did — made the most likely + // production failure completely invisible. + if (e instanceof OrderParseError) { + return NextResponse.json( + { kind: "parse_failed", reason: e.message, messageId: e.messageId }, + { status: 422 } + ); + } + const message = e instanceof Error ? e.message : String(e); return NextResponse.json({ error: message }, { status: 500 }); } diff --git a/src/lib/order-ingestion.ts b/src/lib/order-ingestion.ts index c525367..3b2267f 100644 --- a/src/lib/order-ingestion.ts +++ b/src/lib/order-ingestion.ts @@ -42,9 +42,18 @@ export async function reconcileCardLeg( AND t.transaction_date BETWEEN $2::date - $4::int AND $2::date + $4::int AND (t.description ILIKE '%doordash%' OR t.description ILIKE '%uber%') AND t.amount <= $3::numeric + 0.02 + -- A statement line settles exactly one order. Without this, two orders + -- on the same card inside the window both match the same charge and + -- each books its own credits remainder — double-counting spend. At + -- 10-15 orders a month on one card that is not a corner case. + AND NOT EXISTS ( + SELECT 1 FROM expense_metadata em + WHERE em.matched_transaction_id = t.id + AND ($5::text IS NULL OR em.order_reference IS DISTINCT FROM $5::text) + ) ORDER BY abs(t.amount - $3::numeric), abs(t.transaction_date - $2::date) LIMIT 1`, - [`%${last4}`, day, order.totals.total_charged, windowDays] + [`%${last4}`, day, order.totals.total_charged, windowDays, order.order_reference || null] ); if (!row) return { cardAmount: null, matchedTransactionId: null }; @@ -251,20 +260,29 @@ export async function reconcilePendingOrders(): Promise<{ flags: [], }; - const { cardAmount } = await reconcileCardLeg(probe); + const { cardAmount, matchedTransactionId } = await reconcileCardLeg(probe); if (cardAmount === null) continue; // statement still hasn't arrived const remainder = Number((total - cardAmount).toFixed(2)); let txnId: number | null = null; if (remainder > 0.02 && row.transaction_date >= CUTOVER_DATE) { + // Category from the merchant, never hardcoded. Hardcoding 'dining' here + // silently misfiled every grocery order that arrived with an unstated + // split — reintroducing, through the deferred path, exactly the + // misfiling resolveCategory() exists to prevent. + const category = resolveCategory({ + merchant_name: row.merchant_normalized, + platform: "doordash", + } as ParsedOrder); + const txn = await queryRow<{ id: number }>( `INSERT INTO transactions ( transaction_date, description, amount, amount_aud, category, payment_method, merchant_name, merchant_normalized, transaction_type, owner_id - ) VALUES ($1,$2,$3,$3,'dining','credits',$4,$4,'debit',NULL) + ) VALUES ($1,$2,$3,$3,$5,'credits',$4,$4,'debit',NULL) RETURNING id`, - [row.transaction_date, `Order - ${row.merchant_normalized}`, remainder, row.merchant_normalized] + [row.transaction_date, `Order - ${row.merchant_normalized}`, remainder, row.merchant_normalized, category] ); txnId = txn!.id; created++; @@ -273,10 +291,11 @@ export async function reconcilePendingOrders(): Promise<{ await queryRaw( `UPDATE expense_metadata SET transaction_id = COALESCE($2, transaction_id), + matched_transaction_id = $4, reconciled_at = NOW(), flags = flags || $3::jsonb WHERE id = $1`, - [row.id, txnId, JSON.stringify([`card_leg_${cardAmount.toFixed(2)}`])] + [row.id, txnId, JSON.stringify([`card_leg_${cardAmount.toFixed(2)}`]), matchedTransactionId] ); resolved++; } diff --git a/src/lib/order-parse.ts b/src/lib/order-parse.ts index db03e13..8dec904 100644 --- a/src/lib/order-parse.ts +++ b/src/lib/order-parse.ts @@ -61,6 +61,23 @@ export interface MessageMeta { sender?: string; } +/** + * The message is not a receipt at all — a promotion, a delivery update, an + * adjustment or refund notice. Expected traffic. Skipping it is correct and + * must not raise an alert, or the channel becomes noise and gets ignored. + */ +export class NotAReceiptError extends Error { + constructor(message: string, readonly messageId?: string) { + super(message); + this.name = "NotAReceiptError"; + } +} + +/** + * The message IS a receipt and could not be parsed. This is the failure that + * matters: a provider template change breaks every order at once, silently, and + * the only symptom is spend quietly ceasing to appear. It must alert loudly. + */ export class OrderParseError extends Error { constructor(message: string, readonly messageId?: string) { super(message); @@ -81,6 +98,15 @@ const decodeEntities = (s: string) => const collapse = (s: string) => s.replace(/\s+/g, " ").trim(); +/** URL-decodes without throwing on malformed percent-escapes. */ +function safeDecode(s: string): string { + try { + return decodeURIComponent(s.replace(/%(?![0-9a-f]{2})/gi, "%25")); + } catch { + return s; + } +} + const money = (raw: string): number => Math.abs(parseFloat(raw.replace(/[$,]/g, ""))); /** @@ -136,7 +162,7 @@ function detectPlatform(meta: MessageMeta, html: string): ParsedOrder["platform" } if (/order with Uber Eats/i.test(s)) return "ubereats"; if (/trip with Uber|Uber receipt|Trip fare/i.test(s) || /Trip fare/i.test(html)) return "uber"; - throw new OrderParseError(`cannot determine platform from subject: ${s}`, meta.messageId); + throw new NotAReceiptError(`cannot determine platform from subject: ${s}`, meta.messageId); } function parseMerchant(platform: string, meta: MessageMeta, text: string): string { @@ -194,7 +220,7 @@ function parsePayment(platform: string, html: string, text: string): PaymentBrea // match would record money that never left the account, so drop the failed // attempts before reading any instrument. text = text.replace( - /(?:Visa|MasterCard|Amex|American Express|Uber Cash|Payments)?[^.]{0,60}?[\d,]+\.\d{2}\s+\S+\s+\S+\s*(?:am|pm)?\s*Failed/gi, + /(?:Visa|MasterCard|Amex|American Express|Uber Cash)[^.]{0,40}?[\d,]+\.\d{2}\s+\S+\s+\S+\s*(?:am|pm)?\s*Failed/gi, " " ); @@ -215,7 +241,7 @@ function parsePayment(platform: string, html: string, text: string): PaymentBrea export function parseOrderHTML(html: string, meta: MessageMeta): ParsedOrder { if (!html || html.length < 200) { - throw new OrderParseError("body too short to be a receipt", meta.messageId); + throw new NotAReceiptError("body too short to be a receipt", meta.messageId); } const clean = html.replace(//g, ""); const text = collapse(decodeEntities(stripTags(clean))); @@ -233,14 +259,14 @@ export function parseOrderHTML(html: string, meta: MessageMeta): ParsedOrder { // order. Amendment handling is not in this pass — reject loudly so none is // silently double-counted. if (/We adjusted the total|Your refund has been applied|Previous total/i.test(text)) { - throw new OrderParseError( + throw new NotAReceiptError( "refund/total-adjustment notice — amends an existing order, not a new receipt", meta.messageId ); } if (/There are adjustments to your order/i.test(text)) { - throw new OrderParseError( + throw new NotAReceiptError( "order-adjustment notice, not a receipt — no final total stated", meta.messageId ); @@ -253,18 +279,41 @@ export function parseOrderHTML(html: string, meta: MessageMeta): ParsedOrder { // Uber embeds a real order UUID in the body. DoorDash embeds no order id at // all, so the provider message id is the only stable identity available — // which is correct for ingestion idempotency (one receipt = one order). + // + // Anchor on Uber's own `tripReference` cell — a hidden + // xid present in all 29 captured + // receipts. For ue-00 it equals the UUID the PDF redirect resolves to + // (ubereats.com/orders/34d6b4ee-...), so it is the order's real identity. + // + // The alternative, "first UUID in the document", is positional rather than + // semantic: 4 of 29 receipts carry several UUIDs, and if a template reshuffle + // ever put a per-send tracking id first, the symptom would be a reference + // that changes every fetch and silently duplicates every order on every + // backfill. Fall back to it only when the anchor is absent, and flag when + // that fallback is genuinely ambiguous. let order_reference: string; - const uuid = clean.match( - /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i + const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i; + const anchored = clean.match( + /tripReference[^>]*>\s*xid([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i ); - if (platform !== "doordash" && uuid) { - order_reference = uuid[0].toLowerCase(); + const firstUuid = clean.match(UUID_RE); + if (platform !== "doordash" && (anchored || firstUuid)) { + order_reference = (anchored ? anchored[1] : firstUuid![0]).toLowerCase(); + if (!anchored) { + const distinct = new Set( + (clean.match(new RegExp(UUID_RE.source, "gi")) || []).map((u) => u.toLowerCase()) + ); + // Only ambiguous when there is more than one candidate to choose between. + if (distinct.size > 1) flags.push("order_uuid_ambiguous"); + } } else { + // DoorDash carries no order id anywhere in the receipt, so the provider + // message id is the only stable identity available. That is correct for + // ingestion idempotency: one receipt is one order. if (!meta.messageId) { throw new OrderParseError("no order id in body and no messageId supplied"); } order_reference = `msg:${meta.messageId}`; - if (platform !== "doordash") flags.push("no_order_uuid_fell_back_to_message_id"); } // ---- date ----------------------------------------------------------------