None of these were caught by 105 green tests, because the code they live in was barely tested and the HTTP path was not tested at all. 1. reconcilePendingOrders hardcoded category 'dining', so any order resolved through the deferred path booked as dining regardless of merchant — a Woolworths grocery order that parks and later reconciles was misfiled. That reintroduced, through the back door, exactly the misfiling resolveCategory() exists to prevent. Now calls it. 2. reconcileCardLeg never marked a statement line as consumed, so two orders on the same card inside the +/-4 day window both bound to the same charge and each booked its own credits remainder — double-counting spend. At 10-15 orders a month on one card that is not a corner case. Migration 0020 adds matched_transaction_id with a unique index; the matcher now excludes lines already claimed. 3. The ingest API returned HTTP 200 for every parse failure, and the Slack alert fires only on non-200. So the single most likely production failure — a provider template change breaking every order at once — was completely silent. Split into NotAReceiptError (promotions, delivery updates, refund and adjustment notices: 200, silent, expected traffic) and OrderParseError (it IS a receipt and would not parse: 422, alerts). Also: order_reference now anchors on Uber's own tripReference cell rather than 'first UUID in the document'. I had claimed to verify that the first UUID was always the order UUID; that check compared against zero samples and was vacuous. tripReference is present in all 29 captured receipts and, for ue-00, equals the UUID the PDF redirect resolves to. The positional fallback remains but only flags when there is genuine ambiguity. Adds the API route's first tests — auth gate and error taxonomy — plus anchoring regressions. 63 unit + 53 integration green on five consecutive runs; corpus holds at 63/65.
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
import { describe, it, expect, beforeAll } from "vitest";
|
||||
import { readFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
import { queryRaw } from "../../lib/db";
|
||||
|
||||
/**
|
||||
* The HTTP path had no tests at all — which is how three defects reached the
|
||||
* branch through a suite of 105 green ones. These exercise the route handler
|
||||
* directly (no server needed) so the auth gate and the error taxonomy are
|
||||
* actually covered.
|
||||
*/
|
||||
const dir = resolve(__dirname, "../fixtures/orders/real");
|
||||
const html = (f: string) => readFileSync(resolve(dir, `${f}.html`), "utf-8");
|
||||
|
||||
const TOKEN = "test-ingest-token";
|
||||
let POST: any;
|
||||
|
||||
const req = (body: unknown, token: string | null = TOKEN) =>
|
||||
({
|
||||
headers: { get: (h: string) => (h === "x-ingest-token" ? token : null) },
|
||||
json: async () => body,
|
||||
}) as any;
|
||||
|
||||
beforeAll(async () => {
|
||||
process.env.ORDER_INGEST_TOKEN = TOKEN;
|
||||
({ POST } = await import("../../app/api/orders/ingest/route"));
|
||||
});
|
||||
|
||||
describe("ingest API — auth", () => {
|
||||
it("rejects a missing token", async () => {
|
||||
const res = await POST(req({ html: "x", meta: {} }, null));
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it("rejects a wrong token", async () => {
|
||||
const res = await POST(req({ html: "x", meta: {} }, "nope"));
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
it("rejects a malformed body", async () => {
|
||||
const res = await POST(req({ html: "only html" }));
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ingest API — error taxonomy", () => {
|
||||
const meta = (over = {}) => ({
|
||||
messageId: `api-${Math.random().toString(36).slice(2)}`,
|
||||
subject: "Order Confirmation for Siddharth from Mad Mex",
|
||||
receivedAt: "2026-07-16T03:34:00Z",
|
||||
sender: "DoorDash Order <no-reply@doordash.com>",
|
||||
...over,
|
||||
});
|
||||
|
||||
it("a non-receipt is 200 and silent — it must not alert", async () => {
|
||||
// A newsletter: real traffic, correctly ignored.
|
||||
const res = await POST(req({ html: html("dd-01"), meta: meta({ subject: "Newsletter", sender: "promo@example.com" }) }));
|
||||
expect(res.status).toBe(200);
|
||||
expect((await res.json()).kind).toBe("skipped");
|
||||
});
|
||||
|
||||
it("an adjustment notice is 200 and silent", async () => {
|
||||
const res = await POST(req({
|
||||
html: html("dd-08"),
|
||||
meta: meta({ subject: "Order Confirmation for Siddharth from ALDI" }),
|
||||
}));
|
||||
expect(res.status).toBe(200);
|
||||
expect((await res.json()).kind).toBe("skipped");
|
||||
});
|
||||
|
||||
it("a receipt that cannot be parsed is 422 so it ALERTS", async () => {
|
||||
// A DoorDash receipt with its totals stripped out — i.e. what a provider
|
||||
// template change looks like. Previously this returned 200 and vanished.
|
||||
const broken = html("dd-01")
|
||||
.replace(/Total Charged/g, "Gesamtbetrag")
|
||||
.replace(/Total:/g, "Summe:");
|
||||
const res = await POST(req({ html: broken, meta: meta() }));
|
||||
expect(res.status).toBe(422);
|
||||
const body = await res.json();
|
||||
expect(body.kind).toBe("parse_failed");
|
||||
expect(body.reason).toMatch(/total/i);
|
||||
});
|
||||
|
||||
it("a refund is routed to the amendment path, not ingestion", async () => {
|
||||
const res = await POST(req({
|
||||
html: html("ue-05"),
|
||||
meta: meta({ subject: "Your Tuesday evening order with Uber Eats", sender: "uber.com" }),
|
||||
dryRun: true,
|
||||
}));
|
||||
expect(res.status).toBe(200);
|
||||
const body = await res.json();
|
||||
expect(body.kind).toBe("amendment");
|
||||
expect(body.amendment.new_total).toBeCloseTo(45.73, 2);
|
||||
});
|
||||
|
||||
it("a good receipt ingests", async () => {
|
||||
await queryRaw(`DELETE FROM expense_metadata WHERE source = 'email'`);
|
||||
await queryRaw(`DELETE FROM transactions WHERE description LIKE 'Order - %'`);
|
||||
const res = await POST(req({ html: html("dd-01"), meta: meta({ messageId: "api-ok-1" }) }));
|
||||
expect(res.status).toBe(200);
|
||||
const body = await res.json();
|
||||
expect(body.kind).toBe("order");
|
||||
expect(body.total).toBe(14.64);
|
||||
expect(body.transactionId).not.toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
parseOrderAmendment,
|
||||
applyOrderAmendment,
|
||||
OrderParseError,
|
||||
NotAReceiptError,
|
||||
type MessageMeta,
|
||||
} from "../../lib/order-ingestion";
|
||||
import { EXCLUDE_NON_SPEND } from "../../lib/analytics-sql";
|
||||
@@ -113,9 +114,12 @@ describe("Order parsing — real receipts", () => {
|
||||
});
|
||||
|
||||
it("rejects an order-adjustment notice rather than booking $0.00", () => {
|
||||
// NotAReceiptError, not OrderParseError: this is expected traffic, so it
|
||||
// must be skipped silently. Only a receipt that fails to parse should
|
||||
// alert — see the ingest API's error taxonomy.
|
||||
expect(() =>
|
||||
parseOrderHTML(html("dd-08"), meta({ subject: "Order Confirmation for Siddharth from ALDI" }))
|
||||
).toThrow(OrderParseError);
|
||||
).toThrow(NotAReceiptError);
|
||||
});
|
||||
|
||||
it("rejects a refund notice rather than inserting a duplicate order", () => {
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
validateOrderTotals,
|
||||
resolveCategory,
|
||||
OrderParseError,
|
||||
NotAReceiptError,
|
||||
type MessageMeta,
|
||||
type ParsedOrder,
|
||||
} from "../../lib/order-ingestion";
|
||||
@@ -125,7 +126,7 @@ describe("resolveCategory", () => {
|
||||
|
||||
describe("parse guards", () => {
|
||||
it("throws on a body too short to be a receipt", () => {
|
||||
expect(() => parseOrderHTML("<html></html>", meta())).toThrow(OrderParseError);
|
||||
expect(() => parseOrderHTML("<html></html>", meta())).toThrow(NotAReceiptError);
|
||||
});
|
||||
|
||||
it("throws rather than inventing a platform", () => {
|
||||
@@ -134,3 +135,32 @@ describe("parse guards", () => {
|
||||
).toThrow(/platform/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe("order_reference anchoring", () => {
|
||||
it("takes Uber's tripReference, not the first UUID in the document", () => {
|
||||
const p = parseOrderHTML(
|
||||
html("ue-00"),
|
||||
meta({ subject: "Your Wednesday afternoon order with Uber Eats", sender: "uber.com" })
|
||||
);
|
||||
// The UUID the PDF redirect actually resolves to for this receipt.
|
||||
expect(p.order_reference).toBe("34d6b4ee-da8f-5029-8d14-bd359617c8e9");
|
||||
expect(p.flags).not.toContain("order_uuid_ambiguous");
|
||||
});
|
||||
|
||||
it("is stable across repeated parses of the same message", () => {
|
||||
const m = meta({ subject: "Your Wednesday afternoon order with Uber Eats", sender: "uber.com" });
|
||||
const a = parseOrderHTML(html("ue-00"), m).order_reference;
|
||||
const b = parseOrderHTML(html("ue-00"), m).order_reference;
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
it("flags ambiguity only when several UUIDs and no anchor", () => {
|
||||
// Strip the anchor from a receipt that carries multiple UUIDs (ue-09 has 6).
|
||||
const stripped = html("ue-09").replace(/tripReference/gi, "notTheAnchor");
|
||||
const p = parseOrderHTML(
|
||||
stripped,
|
||||
meta({ subject: "[Family] Your Sunday evening order with Uber Eats", sender: "uber.com" })
|
||||
);
|
||||
expect(p.flags).toContain("order_uuid_ambiguous");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
applyOrderAmendment,
|
||||
reconcilePendingOrders,
|
||||
OrderParseError,
|
||||
NotAReceiptError,
|
||||
type MessageMeta,
|
||||
} from "@/lib/order-ingestion";
|
||||
|
||||
@@ -82,11 +83,25 @@ export async function POST(req: NextRequest) {
|
||||
...result,
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof OrderParseError) {
|
||||
// Not a receipt (promotion, adjustment notice, delivery update). Expected
|
||||
// traffic — 200 with skipped, so n8n does not treat it as a failure.
|
||||
// Not a receipt: promotions, delivery updates, adjustment and refund
|
||||
// notices. Expected traffic — 200 and silent, or the alert channel fills
|
||||
// with noise and stops being read.
|
||||
if (e instanceof NotAReceiptError) {
|
||||
return NextResponse.json({ kind: "skipped", reason: e.message });
|
||||
}
|
||||
|
||||
// IS a receipt, could not be parsed. This is the failure that matters and
|
||||
// it must be loud: a provider template change breaks every order at once,
|
||||
// and the only other symptom is spend quietly ceasing to appear. Returning
|
||||
// 200 here — as this route originally did — made the most likely
|
||||
// production failure completely invisible.
|
||||
if (e instanceof OrderParseError) {
|
||||
return NextResponse.json(
|
||||
{ kind: "parse_failed", reason: e.message, messageId: e.messageId },
|
||||
{ status: 422 }
|
||||
);
|
||||
}
|
||||
|
||||
const message = e instanceof Error ? e.message : String(e);
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
|
||||
@@ -42,9 +42,18 @@ export async function reconcileCardLeg(
|
||||
AND t.transaction_date BETWEEN $2::date - $4::int AND $2::date + $4::int
|
||||
AND (t.description ILIKE '%doordash%' OR t.description ILIKE '%uber%')
|
||||
AND t.amount <= $3::numeric + 0.02
|
||||
-- A statement line settles exactly one order. Without this, two orders
|
||||
-- on the same card inside the window both match the same charge and
|
||||
-- each books its own credits remainder — double-counting spend. At
|
||||
-- 10-15 orders a month on one card that is not a corner case.
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM expense_metadata em
|
||||
WHERE em.matched_transaction_id = t.id
|
||||
AND ($5::text IS NULL OR em.order_reference IS DISTINCT FROM $5::text)
|
||||
)
|
||||
ORDER BY abs(t.amount - $3::numeric), abs(t.transaction_date - $2::date)
|
||||
LIMIT 1`,
|
||||
[`%${last4}`, day, order.totals.total_charged, windowDays]
|
||||
[`%${last4}`, day, order.totals.total_charged, windowDays, order.order_reference || null]
|
||||
);
|
||||
|
||||
if (!row) return { cardAmount: null, matchedTransactionId: null };
|
||||
@@ -251,20 +260,29 @@ export async function reconcilePendingOrders(): Promise<{
|
||||
flags: [],
|
||||
};
|
||||
|
||||
const { cardAmount } = await reconcileCardLeg(probe);
|
||||
const { cardAmount, matchedTransactionId } = await reconcileCardLeg(probe);
|
||||
if (cardAmount === null) continue; // statement still hasn't arrived
|
||||
|
||||
const remainder = Number((total - cardAmount).toFixed(2));
|
||||
let txnId: number | null = null;
|
||||
|
||||
if (remainder > 0.02 && row.transaction_date >= CUTOVER_DATE) {
|
||||
// Category from the merchant, never hardcoded. Hardcoding 'dining' here
|
||||
// silently misfiled every grocery order that arrived with an unstated
|
||||
// split — reintroducing, through the deferred path, exactly the
|
||||
// misfiling resolveCategory() exists to prevent.
|
||||
const category = resolveCategory({
|
||||
merchant_name: row.merchant_normalized,
|
||||
platform: "doordash",
|
||||
} as ParsedOrder);
|
||||
|
||||
const txn = await queryRow<{ id: number }>(
|
||||
`INSERT INTO transactions (
|
||||
transaction_date, description, amount, amount_aud, category,
|
||||
payment_method, merchant_name, merchant_normalized, transaction_type, owner_id
|
||||
) VALUES ($1,$2,$3,$3,'dining','credits',$4,$4,'debit',NULL)
|
||||
) VALUES ($1,$2,$3,$3,$5,'credits',$4,$4,'debit',NULL)
|
||||
RETURNING id`,
|
||||
[row.transaction_date, `Order - ${row.merchant_normalized}`, remainder, row.merchant_normalized]
|
||||
[row.transaction_date, `Order - ${row.merchant_normalized}`, remainder, row.merchant_normalized, category]
|
||||
);
|
||||
txnId = txn!.id;
|
||||
created++;
|
||||
@@ -273,10 +291,11 @@ export async function reconcilePendingOrders(): Promise<{
|
||||
await queryRaw(
|
||||
`UPDATE expense_metadata
|
||||
SET transaction_id = COALESCE($2, transaction_id),
|
||||
matched_transaction_id = $4,
|
||||
reconciled_at = NOW(),
|
||||
flags = flags || $3::jsonb
|
||||
WHERE id = $1`,
|
||||
[row.id, txnId, JSON.stringify([`card_leg_${cardAmount.toFixed(2)}`])]
|
||||
[row.id, txnId, JSON.stringify([`card_leg_${cardAmount.toFixed(2)}`]), matchedTransactionId]
|
||||
);
|
||||
resolved++;
|
||||
}
|
||||
|
||||
+59
-10
@@ -61,6 +61,23 @@ export interface MessageMeta {
|
||||
sender?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The message is not a receipt at all — a promotion, a delivery update, an
|
||||
* adjustment or refund notice. Expected traffic. Skipping it is correct and
|
||||
* must not raise an alert, or the channel becomes noise and gets ignored.
|
||||
*/
|
||||
export class NotAReceiptError extends Error {
|
||||
constructor(message: string, readonly messageId?: string) {
|
||||
super(message);
|
||||
this.name = "NotAReceiptError";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The message IS a receipt and could not be parsed. This is the failure that
|
||||
* matters: a provider template change breaks every order at once, silently, and
|
||||
* the only symptom is spend quietly ceasing to appear. It must alert loudly.
|
||||
*/
|
||||
export class OrderParseError extends Error {
|
||||
constructor(message: string, readonly messageId?: string) {
|
||||
super(message);
|
||||
@@ -81,6 +98,15 @@ const decodeEntities = (s: string) =>
|
||||
|
||||
const collapse = (s: string) => s.replace(/\s+/g, " ").trim();
|
||||
|
||||
/** URL-decodes without throwing on malformed percent-escapes. */
|
||||
function safeDecode(s: string): string {
|
||||
try {
|
||||
return decodeURIComponent(s.replace(/%(?![0-9a-f]{2})/gi, "%25"));
|
||||
} catch {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
|
||||
const money = (raw: string): number => Math.abs(parseFloat(raw.replace(/[$,]/g, "")));
|
||||
|
||||
/**
|
||||
@@ -136,7 +162,7 @@ function detectPlatform(meta: MessageMeta, html: string): ParsedOrder["platform"
|
||||
}
|
||||
if (/order with Uber Eats/i.test(s)) return "ubereats";
|
||||
if (/trip with Uber|Uber receipt|Trip fare/i.test(s) || /Trip fare/i.test(html)) return "uber";
|
||||
throw new OrderParseError(`cannot determine platform from subject: ${s}`, meta.messageId);
|
||||
throw new NotAReceiptError(`cannot determine platform from subject: ${s}`, meta.messageId);
|
||||
}
|
||||
|
||||
function parseMerchant(platform: string, meta: MessageMeta, text: string): string {
|
||||
@@ -194,7 +220,7 @@ function parsePayment(platform: string, html: string, text: string): PaymentBrea
|
||||
// match would record money that never left the account, so drop the failed
|
||||
// attempts before reading any instrument.
|
||||
text = text.replace(
|
||||
/(?:Visa|MasterCard|Amex|American Express|Uber Cash|Payments)?[^.]{0,60}?[\d,]+\.\d{2}\s+\S+\s+\S+\s*(?:am|pm)?\s*Failed/gi,
|
||||
/(?:Visa|MasterCard|Amex|American Express|Uber Cash)[^.]{0,40}?[\d,]+\.\d{2}\s+\S+\s+\S+\s*(?:am|pm)?\s*Failed/gi,
|
||||
" "
|
||||
);
|
||||
|
||||
@@ -215,7 +241,7 @@ function parsePayment(platform: string, html: string, text: string): PaymentBrea
|
||||
|
||||
export function parseOrderHTML(html: string, meta: MessageMeta): ParsedOrder {
|
||||
if (!html || html.length < 200) {
|
||||
throw new OrderParseError("body too short to be a receipt", meta.messageId);
|
||||
throw new NotAReceiptError("body too short to be a receipt", meta.messageId);
|
||||
}
|
||||
const clean = html.replace(/<!--[\s\S]*?-->/g, "");
|
||||
const text = collapse(decodeEntities(stripTags(clean)));
|
||||
@@ -233,14 +259,14 @@ export function parseOrderHTML(html: string, meta: MessageMeta): ParsedOrder {
|
||||
// order. Amendment handling is not in this pass — reject loudly so none is
|
||||
// silently double-counted.
|
||||
if (/We adjusted the total|Your refund has been applied|Previous total/i.test(text)) {
|
||||
throw new OrderParseError(
|
||||
throw new NotAReceiptError(
|
||||
"refund/total-adjustment notice — amends an existing order, not a new receipt",
|
||||
meta.messageId
|
||||
);
|
||||
}
|
||||
|
||||
if (/There are adjustments to your order/i.test(text)) {
|
||||
throw new OrderParseError(
|
||||
throw new NotAReceiptError(
|
||||
"order-adjustment notice, not a receipt — no final total stated",
|
||||
meta.messageId
|
||||
);
|
||||
@@ -253,18 +279,41 @@ export function parseOrderHTML(html: string, meta: MessageMeta): ParsedOrder {
|
||||
// Uber embeds a real order UUID in the body. DoorDash embeds no order id at
|
||||
// all, so the provider message id is the only stable identity available —
|
||||
// which is correct for ingestion idempotency (one receipt = one order).
|
||||
//
|
||||
// Anchor on Uber's own `tripReference` cell — a hidden
|
||||
// <td class="tripReference">xid<UUID></td> present in all 29 captured
|
||||
// receipts. For ue-00 it equals the UUID the PDF redirect resolves to
|
||||
// (ubereats.com/orders/34d6b4ee-...), so it is the order's real identity.
|
||||
//
|
||||
// The alternative, "first UUID in the document", is positional rather than
|
||||
// semantic: 4 of 29 receipts carry several UUIDs, and if a template reshuffle
|
||||
// ever put a per-send tracking id first, the symptom would be a reference
|
||||
// that changes every fetch and silently duplicates every order on every
|
||||
// backfill. Fall back to it only when the anchor is absent, and flag when
|
||||
// that fallback is genuinely ambiguous.
|
||||
let order_reference: string;
|
||||
const uuid = clean.match(
|
||||
/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i
|
||||
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;
|
||||
const anchored = clean.match(
|
||||
/tripReference[^>]*>\s*xid([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i
|
||||
);
|
||||
if (platform !== "doordash" && uuid) {
|
||||
order_reference = uuid[0].toLowerCase();
|
||||
const firstUuid = clean.match(UUID_RE);
|
||||
if (platform !== "doordash" && (anchored || firstUuid)) {
|
||||
order_reference = (anchored ? anchored[1] : firstUuid![0]).toLowerCase();
|
||||
if (!anchored) {
|
||||
const distinct = new Set(
|
||||
(clean.match(new RegExp(UUID_RE.source, "gi")) || []).map((u) => u.toLowerCase())
|
||||
);
|
||||
// Only ambiguous when there is more than one candidate to choose between.
|
||||
if (distinct.size > 1) flags.push("order_uuid_ambiguous");
|
||||
}
|
||||
} else {
|
||||
// DoorDash carries no order id anywhere in the receipt, so the provider
|
||||
// message id is the only stable identity available. That is correct for
|
||||
// ingestion idempotency: one receipt is one order.
|
||||
if (!meta.messageId) {
|
||||
throw new OrderParseError("no order id in body and no messageId supplied");
|
||||
}
|
||||
order_reference = `msg:${meta.messageId}`;
|
||||
if (platform !== "doordash") flags.push("no_order_uuid_fell_back_to_message_id");
|
||||
}
|
||||
|
||||
// ---- date ----------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user