feat(orders): rebuild the receipt parser against real captured email

The previous parser was written against synthetic fixtures shaped to match the code. It invented a table layout DoorDash does not send, generated order_reference from Math.random(), read the order date from a 'Date:' string present in no real message, and detected [Family] by searching the body for the substring 'family'. Its tests passed because the fixtures were built to satisfy it. Against 36 real DoorDash and 29 real Uber Eats receipts it does not work. Rebuilt from the real corpus. 63 of 65 now parse and validate; the 2 rejected are correctly rejected — one is an order-adjustment notice and one a refund, neither of which is a receipt. Corrects an inherited diagnosis: the Mad Mex 'Discounts -$24.09' was recorded as an HTML-flattening artefact masking a 'true discount of $9.45'. Parsing the table cells structurally returns the same figures and no $9.45 exists anywhere in the message — DoorDash genuinely prints a Discounts line that equals subtotal plus service fee, and the components fail to reconcile on 32 of 36 receipts. So the breakdown is stored as provenance and never gated on; validation instead cross-checks the two independently stated totals and the payment line, which is the number that becomes money. Real-world cases the corpus forced, none of which were in the spec: [Family] orders are LKR purchases for family in Sri Lanka (reading them as dollars inflates ~200x), Swiss orders arrive in CHF, grocery 'Final receipt' mails carry no Total Charged row, and a declined payment is printed alongside the successful retry and must be skipped or it records money that never moved. order_reference now comes from the Uber order UUID embedded in the body, or the provider message id where DoorDash supplies no order id at all — never random, so re-ingestion is genuinely idempotent.
2026-07-26 22:25:26 +10:00
parent 6d3b6e1a9d
commit 33db7d05ef
12 changed files with 4714 additions and 0 deletions
@@ -0,0 +1,433 @@
+/**
+ * Order receipt parsing, written against REAL captured emails.
+ *
+ * History: the first version of this parser was written against synthetic
+ * fixtures that were shaped to match the code rather than the mail. It invented
+ * a <table><tr><td>Label</td><td>$X</td></tr> layout that DoorDash does not
+ * send, derived order_reference from Math.random(), and read the order date
+ * from a `Date: YYYY-MM-DD` string that appears in no real message. All of it
+ * passed its tests. This version is built from 36 real DoorDash and 29 real
+ * Uber Eats receipts; see docs in memory-bank/order-ingestion-implementation.md.
+ *
+ * Governing rule: parse or throw. Never fabricate a value that the mail did not
+ * state (I9). A caller that gets a ParsedOrder back can trust every field in it.
+ */
+
+export interface LineItem {
+  qty: number;
+  description: string;
+  amount: number;
+  options?: string[];
+}
+
+export interface PaymentBreakdown {
+  credits_amount: number | null;
+  card_amount: number | null;
+  card_last4: string | null;
+  /** true when the mail states a payment method it does not fully disaggregate. */
+  ambiguous: boolean;
+}
+
+export interface OrderTotals {
+  subtotal: number | null;
+  taxes: number | null;
+  delivery_fee: number | null;
+  service_fee: number | null;
+  tip: number | null;
+  discounts: number | null;
+  total_charged: number;
+}
+
+export interface ParsedOrder {
+  order_reference: string;
+  platform: "doordash" | "ubereats" | "uber";
+  merchant_name: string;
+  order_datetime: string;
+  currency: string;
+  payment: PaymentBreakdown;
+  totals: OrderTotals;
+  line_items: LineItem[];
+  is_family: boolean;
+  flags: string[];
+}
+
+/** Everything the parser needs that lives on the message, not in the body. */
+export interface MessageMeta {
+  /** Provider message id. The only stable per-mail identity DoorDash offers. */
+  messageId: string;
+  subject: string;
+  /** ISO 8601. The authoritative order date — the body carries no reliable one. */
+  receivedAt: string;
+  sender?: string;
+}
+
+export class OrderParseError extends Error {
+  constructor(message: string, readonly messageId?: string) {
+    super(message);
+    this.name = "OrderParseError";
+  }
+}
+
+const stripTags = (s: string) => s.replace(/<[^>]+>/g, " ");
+
+const decodeEntities = (s: string) =>
+  s
+    .replace(/&nbsp;/gi, " ")
+    .replace(/&amp;/gi, "&")
+    .replace(/&#39;|&apos;/gi, "'")
+    .replace(/&quot;/gi, '"')
+    .replace(/&#36;/g, "$")
+    .replace(/&hellip;/gi, "…");
+
+const collapse = (s: string) => s.replace(/\s+/g, " ").trim();
+
+const money = (raw: string): number => Math.abs(parseFloat(raw.replace(/[$,]/g, "")));
+
+/**
+ * DoorDash renders each total as its own nested table:
+ *   <td align="left" ...>Subtotal</td> <!----> <td align="right" ...>$22.10</td>
+ * Reading the pair structurally is what stops the label/value rebinding that
+ * flattening causes (I9) — flattened, "Discounts -$24.09 Total Charged $14.64"
+ * invites a regex to bind the wrong number to the wrong label.
+ */
+function tdPairValue(html: string, label: string): number | null {
+  const re = new RegExp(
+    `<td[^>]*>\\s*${label}\\s*</td>\\s*<td[^>]*>\\s*(-?\\s*\\$?[\\d,]+\\.\\d{2})\\s*</td>`,
+    "i"
+  );
+  const m = html.match(re);
+  return m ? money(m[1]) : null;
+}
+
+function parseDoorDashLineItems(html: string): LineItem[] {
+  // <td width="10%">1x</td><td width="75%"><b>Name</b> (Cat)<br><font>• Opt</font>…</td><td width="15%">$22.10</td>
+  const re =
+    /<td[^>]*width="10%"[^>]*>\s*(\d+)x\s*<\/td>\s*<td[^>]*width="75%"[^>]*>([\s\S]*?)<\/td>\s*<td[^>]*width="15%"[^>]*>\s*\$?([\d,]+\.\d{2})\s*<\/td>/gi;
+  const items: LineItem[] = [];
+  for (const m of html.matchAll(re)) {
+    const parts = decodeEntities(stripTags(m[2]))
+      .split("•")
+      .map((p) => collapse(p))
+      .filter(Boolean);
+    if (!parts.length) continue;
+    items.push({
+      qty: parseInt(m[1], 10),
+      description: parts[0],
+      amount: money(m[3]),
+      options: parts.slice(1),
+    });
+  }
+  return items;
+}
+
+/** Reads "<Label> [CUR] 1,234.56" out of already-flattened Uber receipt text. */
+function extractLabelled(text: string, label: string): number | null {
+  const m = text.match(
+    new RegExp(`${label}\\s*(?:[A-Z]{3})?\\s*\\$?\\s*([\\d,]+\\.\\d{2})`, "i")
+  );
+  return m ? money(m[1]) : null;
+}
+
+function detectPlatform(meta: MessageMeta, html: string): ParsedOrder["platform"] {
+  const s = meta.subject || "";
+  const from = (meta.sender || "").toLowerCase();
+  if (/doordash/i.test(from) || /^\s*(\[Family\]\s*)?Order Confirmation for/i.test(s)) {
+    return "doordash";
+  }
+  if (/order with Uber Eats/i.test(s)) return "ubereats";
+  if (/trip with Uber|Uber receipt|Trip fare/i.test(s) || /Trip fare/i.test(html)) return "uber";
+  throw new OrderParseError(`cannot determine platform from subject: ${s}`, meta.messageId);
+}
+
+function parseMerchant(platform: string, meta: MessageMeta, text: string): string {
+  if (platform === "doordash") {
+    // Subject: "Order Confirmation for Siddharth from Mad Mex"
+    const m = meta.subject.match(/Order Confirmation for \S+\s+from\s+([\s\S]+?)\s*$/i);
+    if (m) return collapse(m[1]);
+  }
+  if (platform === "ubereats") {
+    // Body: "Here's your receipt for TEG Kebabs & Biryani."
+    const m = text.match(/receipt for\s+([\s\S]+?)\s*\.\s/i);
+    if (m) return collapse(m[1]);
+  }
+  if (platform === "uber") return "Uber Trip";
+  throw new OrderParseError(`cannot determine merchant`, meta.messageId);
+}
+
+function parsePayment(platform: string, html: string, text: string): PaymentBreakdown {
+  // eslint-disable-next-line no-param-reassign
+  const out: PaymentBreakdown = {
+    credits_amount: null,
+    card_amount: null,
+    card_last4: null,
+    ambiguous: false,
+  };
+
+  if (platform === "doordash") {
+    const paid = text.match(/Paid with\s+([^\n]{1,120}?)(?:\s{2,}|$)/i);
+    const line = paid ? collapse(paid[1]) : "";
+
+    // "MasterCard Ending in 8032 and/or credits" — DoorDash names both methods
+    // and never the split. Resolved as card (user, 2026-07-26: the real charge
+    // went to 8032), which is also the conservative reading: a card order
+    // creates no transaction, so it cannot double-count against the statement
+    // line that will arrive for that card. Flagged either way.
+    if (/and\/or/i.test(line)) {
+      out.ambiguous = true;
+      const l4 = line.match(/Ending in\s*(\d{3,4})/i);
+      out.card_last4 = l4 ? l4[1] : null;
+      return out;
+    }
+    if (/credits/i.test(line)) return out; // credits-only; amount filled from total
+    const l4 = line.match(/Ending in\s*(\d{3,4})/i);
+    if (l4) {
+      out.card_last4 = l4[1];
+      return out; // card-only; amount filled from total
+    }
+    return out;
+  }
+
+  // Uber Eats / Uber: "Payments Uber Cash $25.33", or a card line, or — on
+  // [Family] orders placed on a shared account — just the payer's name:
+  // "Payments Siddharth LKR 3,783.20". That last form names no instrument at
+  // all, so the split cannot be read from the mail and must not be invented.
+  // A declined attempt is still printed, immediately followed by "Failed":
+  //   Visa ••••8841 LKR 4,267.01 7/5/26 7:20 pm Failed
+  //   Siddharth     LKR 3,757.01 7/5/26 9:02 pm
+  // (grocery order re-charged lower after sold-out items). Taking the first
+  // match would record money that never left the account, so drop the failed
+  // attempts before reading any instrument.
+  text = text.replace(
+    /(?:Visa|MasterCard|Amex|American Express|Uber Cash|Payments)?[^.]{0,60}?[\d,]+\.\d{2}\s+\S+\s+\S+\s*(?:am|pm)?\s*Failed/gi,
+    " "
+  );
+
+  const cash = text.match(/Uber Cash\s*(?:[A-Z]{3})?\s*\$?([\d,]+\.\d{2})/i);
+  if (cash) out.credits_amount = money(cash[1]);
+  const card = text.match(
+    /(?:Visa|MasterCard|American Express|Amex)[^\d]*(\d{4})[^\d]*(?:[A-Z]{3})?\s*\$?([\d,]+\.\d{2})/i
+  );
+  if (card) {
+    out.card_last4 = card[1];
+    out.card_amount = money(card[2]);
+  }
+  if (!cash && !card && /Payments\s+\S+\s+(?:[A-Z]{3}\s|\$)/.test(text)) {
+    out.ambiguous = true;
+  }
+  return out;
+}
+
+export function parseOrderHTML(html: string, meta: MessageMeta): ParsedOrder {
+  if (!html || html.length < 200) {
+    throw new OrderParseError("body too short to be a receipt", meta.messageId);
+  }
+  const clean = html.replace(/<!--[\s\S]*?-->/g, "");
+  const text = collapse(decodeEntities(stripTags(clean)));
+  const flags: string[] = [];
+  let explicitCurrency: string | null = null;
+
+  // Grocery orders (ALDI, Woolworths) generate a follow-up "There are
+  // adjustments to your order" mail for out-of-stock and substituted items. It
+  // reuses the receipt layout but states `Total: $0.00` — the real amount is
+  // settled later. Ingesting it would book a $0.00 order and, worse, its
+  // order_reference would collide with nothing and create a phantom row.
+  // Refund / total-adjustment mails ("We adjusted the total for your recent
+  // order", "Previous total ... Refund ... New Total"). These restate an order
+  // already ingested and must be applied as an amendment, not inserted as a new
+  // order. Amendment handling is not in this pass — reject loudly so none is
+  // silently double-counted.
+  if (/We adjusted the total|Your refund has been applied|Previous total/i.test(text)) {
+    throw new OrderParseError(
+      "refund/total-adjustment notice — amends an existing order, not a new receipt",
+      meta.messageId
+    );
+  }
+
+  if (/There are adjustments to your order/i.test(text)) {
+    throw new OrderParseError(
+      "order-adjustment notice, not a receipt — no final total stated",
+      meta.messageId
+    );
+  }
+
+  const platform = detectPlatform(meta, text);
+  const merchant_name = parseMerchant(platform, meta, text);
+
+  // ---- order_reference -----------------------------------------------------
+  // Uber embeds a real order UUID in the body. DoorDash embeds no order id at
+  // all, so the provider message id is the only stable identity available —
+  // which is correct for ingestion idempotency (one receipt = one order).
+  let order_reference: string;
+  const uuid = clean.match(
+    /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i
+  );
+  if (platform !== "doordash" && uuid) {
+    order_reference = uuid[0].toLowerCase();
+  } else {
+    if (!meta.messageId) {
+      throw new OrderParseError("no order id in body and no messageId supplied");
+    }
+    order_reference = `msg:${meta.messageId}`;
+    if (platform !== "doordash") flags.push("no_order_uuid_fell_back_to_message_id");
+  }
+
+  // ---- date ----------------------------------------------------------------
+  // From the message, never the body. DoorDash confirmations are sent at order
+  // time; the body's own date strings are inconsistent and locale-formatted.
+  const received = new Date(meta.receivedAt);
+  if (isNaN(received.getTime())) {
+    throw new OrderParseError(`unparseable receivedAt: ${meta.receivedAt}`, meta.messageId);
+  }
+  const order_datetime = received.toISOString();
+
+  // ---- [Family] ------------------------------------------------------------
+  // A subject prefix on Uber Eats: "[Family] Your Sunday evening order with…".
+  // Deliberately anchored — a bare substring search for "family" matches
+  // footer copy and merchant names, and a false positive here silently drops
+  // the order out of every budget.
+  const is_family = /^\s*\[Family\]/i.test(meta.subject || "");
+
+  // ---- totals --------------------------------------------------------------
+  let totals: OrderTotals;
+  if (platform === "doordash") {
+    // Grocery "Final receipt" mails price each item and carry no Total Charged
+    // row; the only stated total is the header. Fall back to it explicitly
+    // rather than letting a partial total through.
+    const headerTotal = text.match(/Total:\s*\$?([\d,]+\.\d{2})/i);
+    const total =
+      tdPairValue(clean, "Total Charged") ??
+      tdPairValue(clean, "Total") ??
+      (headerTotal ? money(headerTotal[1]) : null);
+    if (total === null) {
+      throw new OrderParseError("no total stated anywhere in receipt", meta.messageId);
+    }
+    totals = {
+      subtotal: tdPairValue(clean, "Subtotal"),
+      taxes: tdPairValue(clean, "Taxes"),
+      delivery_fee: tdPairValue(clean, "Delivery Fee"),
+      service_fee: tdPairValue(clean, "Service Fee"),
+      tip: tdPairValue(clean, "Tip"),
+      discounts: tdPairValue(clean, "Discounts"),
+      total_charged: total,
+    };
+  } else {
+    // Uber Eats states a Total, optionally in a foreign currency:
+    //   "Total $25.33"  |  "Total LKR 3,783.20"
+    // The [Family] orders are placed for family in Sri Lanka and are priced in
+    // LKR — reading those as dollars would inflate them ~200x, which is a large
+    // part of why they must not reach a budget untagged.
+    const m = text.match(/(?:New Total|Total)\s*(?:([A-Z]{3})\s*)?\$?\s*([\d,]+\.\d{2})/);
+    if (!m) throw new OrderParseError("no Total found", meta.messageId);
+    totals = {
+      subtotal: extractLabelled(text, "Item subtotal"),
+      taxes: extractLabelled(text, "Tax"),
+      delivery_fee: extractLabelled(text, "Delivery Fee"),
+      service_fee: extractLabelled(text, "Service Fee"),
+      tip: null,
+      discounts: null,
+      total_charged: money(m[2]),
+    };
+    if (m[1]) explicitCurrency = m[1].toUpperCase().replace(/\$$/, "");
+  }
+
+  // ---- payment -------------------------------------------------------------
+  const payment = parsePayment(platform, clean, text);
+  if (payment.ambiguous) {
+    // Settled as card — see parsePayment. No credits transaction is created.
+    flags.push("payment_split_not_stated_settled_as_card");
+    payment.card_amount = totals.total_charged;
+  } else if (platform === "doordash") {
+    // DoorDash names the method but not the amount; the total is the amount.
+    if (payment.card_last4) payment.card_amount = totals.total_charged;
+    else payment.credits_amount = totals.total_charged;
+  }
+
+  // ---- line items ----------------------------------------------------------
+  // Uber Eats receipts carry no itemisation (verified across 29 real mails).
+  const line_items =
+    platform === "doordash" ? parseDoorDashLineItems(clean) : [];
+  if (platform === "doordash" && line_items.length === 0) {
+    flags.push("no_line_items_parsed");
+  }
+
+  const currency =
+    explicitCurrency ||
+    (/\b(NZD|USD|LKR|CHF|EUR|GBP|SGD|INR)\b/.test(text)
+      ? (text.match(/\b(NZD|USD|LKR|CHF|EUR|GBP|SGD|INR)\b/) as RegExpMatchArray)[1]
+      : "AUD");
+
+  return {
+    order_reference,
+    platform,
+    merchant_name,
+    order_datetime,
+    currency,
+    payment,
+    totals,
+    line_items,
+    is_family,
+    flags,
+  };
+}
+
+/**
+ * Integrity check on the amount we are about to record as spend.
+ *
+ * NOT an arithmetic reconciliation of the fee breakdown. Measured against 36
+ * real DoorDash receipts, the components do not sum to the total on 32 of them:
+ * DoorDash's `Discounts` line frequently equals subtotal + service fee exactly
+ * (Mad Mex: subtotal 22.10, service 1.99, Discounts -24.09, Total Charged
+ * 14.64) and sometimes differs by an unrelated margin. Whatever that line means
+ * to DoorDash, it is not a term in `total = components`.
+ *
+ * This corrects an earlier diagnosis that read the same numbers as an
+ * HTML-flattening artefact with a "true discount of $9.45". Parsing the table
+ * cells structurally yields the identical figures, and no $9.45 appears
+ * anywhere in the message — the receipt genuinely says this.
+ *
+ * So the breakdown is stored as provenance and never gated on. What IS checked
+ * is the number that becomes money in the ledger: DoorDash states the total
+ * twice, independently (a `Total: $X` header and a `Total Charged` table row),
+ * and those must agree. That catches a mis-parse, which is the failure that
+ * actually matters.
+ */
+export function validateOrderTotals(
+  order: ParsedOrder,
+  html?: string
+): { ok: boolean; reason?: string } {
+  const t = order.totals;
+
+  if (!(t.total_charged > 0)) {
+    return { ok: false, reason: `non-positive total ${t.total_charged}` };
+  }
+
+  // Cross-check the header total against the table total where both exist.
+  if (html && order.platform === "doordash") {
+    const header = collapse(decodeEntities(stripTags(html))).match(
+      /Total:\s*\$?([\d,]+\.\d{2})/i
+    );
+    if (header) {
+      const stated = money(header[1]);
+      if (Math.abs(stated - t.total_charged) > 0.02) {
+        return {
+          ok: false,
+          reason: `header total ${stated.toFixed(2)} disagrees with Total Charged ${t.total_charged.toFixed(2)}`,
+        };
+      }
+    }
+  }
+
+  // The payment line must account for the total, or we are recording an amount
+  // no stated payment method covers.
+  if (!order.payment.ambiguous) {
+    const paid = (order.payment.credits_amount || 0) + (order.payment.card_amount || 0);
+    if (paid > 0 && Math.abs(paid - t.total_charged) > 0.02) {
+      return {
+        ok: false,
+        reason: `payments sum to ${paid.toFixed(2)} but receipt states ${t.total_charged.toFixed(2)}`,
+      };
+    }
+  }
+
+  return { ok: true };
+}