siddharthd
dd0462a5f9
ci / lint-test (push) Successful in 53s
`•` was missing from the entity table, and that was not cosmetic. DoorDash separates an item's name from its options with a bullet and parseDoorDashLineItems splits on the literal "•" — so left encoded, the split never happened and the line collapsed into the description: "Bucket and Side Pack (Meal Deals) • Hot Bucket • Chips" with options []. The entity showed on screen and the structure behind it was gone. Numeric entities are now decoded generically rather than one at a time, which is how $ came to be listed individually while its neighbours were not, and & resolves last so a literal "•" stays as written instead of turning into a bullet. Repaired the 47 stored rows by re-parsing the captured email behind each one rather than string-replacing the entity, since a replacement would have fixed the display and left options [] underneath. Rehearsed first: all 47 re-parsed, all 47 gained options, 0 line items lost, 0 amounts changed. Old values kept in dump/rollback-line-items-20260728-223737.json. Pre-existing — 22 rows predate today — but the pre-cutover backfill more than doubled the affected rows, which is what surfaced it. Verified in the Order details panel, not in SQL.
817 lines
34 KiB
TypeScript
817 lines
34 KiB
TypeScript
/**
|
||
* Order receipt parsing, written against REAL captured emails.
|
||
*
|
||
* History: the first version of this parser was written against synthetic
|
||
* fixtures that were shaped to match the code rather than the mail. It invented
|
||
* a <table><tr><td>Label</td><td>$X</td></tr> layout that DoorDash does not
|
||
* send, derived order_reference from Math.random(), and read the order date
|
||
* from a `Date: YYYY-MM-DD` string that appears in no real message. All of it
|
||
* passed its tests. This version is built from 36 real DoorDash and 29 real
|
||
* Uber Eats receipts; see docs in memory-bank/order-ingestion-implementation.md.
|
||
*
|
||
* Governing rule: parse or throw. Never fabricate a value that the mail did not
|
||
* state (I9). A caller that gets a ParsedOrder back can trust every field in it.
|
||
*/
|
||
|
||
export interface LineItem {
|
||
qty: number;
|
||
description: string;
|
||
amount: number;
|
||
options?: string[];
|
||
}
|
||
|
||
/**
|
||
* A stop on the receipt's map: pick-up, delivery, or (for a trip) the ride's
|
||
* start and end. Uber prints these for every order under `Order details`.
|
||
*/
|
||
export interface RoutePoint {
|
||
/** "Pick-up" / "Delivery" — whatever the receipt itself calls it. */
|
||
label: string;
|
||
/** Local time as printed, e.g. "1:20 pm". No date; the receipt gives none. */
|
||
time: string | null;
|
||
address: string;
|
||
}
|
||
|
||
export interface PaymentBreakdown {
|
||
credits_amount: number | null;
|
||
card_amount: number | null;
|
||
card_last4: string | null;
|
||
/** true when the mail states a payment method it does not fully disaggregate. */
|
||
ambiguous: boolean;
|
||
}
|
||
|
||
export interface OrderTotals {
|
||
subtotal: number | null;
|
||
taxes: number | null;
|
||
delivery_fee: number | null;
|
||
service_fee: number | null;
|
||
tip: number | null;
|
||
discounts: number | null;
|
||
total_charged: number;
|
||
}
|
||
|
||
export interface ParsedOrder {
|
||
order_reference: string;
|
||
platform: "doordash" | "ubereats" | "uber";
|
||
merchant_name: string;
|
||
order_datetime: string;
|
||
currency: string;
|
||
payment: PaymentBreakdown;
|
||
totals: OrderTotals;
|
||
line_items: LineItem[];
|
||
/** Uber only. Empty for DoorDash, whose receipts carry no addresses. */
|
||
route: RoutePoint[];
|
||
is_family: boolean;
|
||
flags: string[];
|
||
}
|
||
|
||
/** Everything the parser needs that lives on the message, not in the body. */
|
||
export interface MessageMeta {
|
||
/** Provider message id. The only stable per-mail identity DoorDash offers. */
|
||
messageId: string;
|
||
subject: string;
|
||
/** ISO 8601. The authoritative order date — the body carries no reliable one. */
|
||
receivedAt: string;
|
||
sender?: string;
|
||
}
|
||
|
||
/**
|
||
* The message is not a receipt at all — a promotion, a delivery update, an
|
||
* adjustment or refund notice. Expected traffic. Skipping it is correct and
|
||
* must not raise an alert, or the channel becomes noise and gets ignored.
|
||
*/
|
||
export class NotAReceiptError extends Error {
|
||
constructor(message: string, readonly messageId?: string) {
|
||
super(message);
|
||
this.name = "NotAReceiptError";
|
||
}
|
||
}
|
||
|
||
/**
|
||
* The message IS a receipt and could not be parsed. This is the failure that
|
||
* matters: a provider template change breaks every order at once, silently, and
|
||
* the only symptom is spend quietly ceasing to appear. It must alert loudly.
|
||
*/
|
||
export class OrderParseError extends Error {
|
||
constructor(message: string, readonly messageId?: string) {
|
||
super(message);
|
||
this.name = "OrderParseError";
|
||
}
|
||
}
|
||
|
||
const stripTags = (s: string) => s.replace(/<[^>]+>/g, " ");
|
||
|
||
/**
|
||
* Entity decoding, ordered so `&` resolves last.
|
||
*
|
||
* `•` was missing, and it is not a cosmetic omission: DoorDash separates
|
||
* an item's name from its options with a bullet, and parseDoorDashLineItems
|
||
* splits on the literal "•". Left encoded, the split never happens and the
|
||
* whole line collapses into the description — "Fire Extinguisher (Chicken
|
||
* Burgers) • Regular" instead of a name plus one option. So the entity
|
||
* showed up on screen AND the structure behind it was lost.
|
||
*
|
||
* Numeric entities are decoded generically rather than one at a time, which is
|
||
* how $ came to be listed individually while its neighbours were not.
|
||
*
|
||
* `&` goes last because decoding it first turns a literal "&bull;" —
|
||
* text that should stay as written — into a bullet.
|
||
*/
|
||
const decodeEntities = (s: string) =>
|
||
s
|
||
.replace(/ /gi, " ")
|
||
.replace(/'|'/gi, "'")
|
||
.replace(/"/gi, '"')
|
||
.replace(/…/gi, "…")
|
||
.replace(/•/gi, "•")
|
||
.replace(/·/gi, "·")
|
||
.replace(/–/gi, "–")
|
||
.replace(/—/gi, "—")
|
||
.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16)))
|
||
.replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))
|
||
.replace(/&/gi, "&");
|
||
|
||
const collapse = (s: string) => s.replace(/\s+/g, " ").trim();
|
||
|
||
const money = (raw: string): number => Math.abs(parseFloat(raw.replace(/[$,]/g, "")));
|
||
|
||
/**
|
||
* DoorDash renders each total as its own nested table:
|
||
* <td align="left" ...>Subtotal</td> <!----> <td align="right" ...>$22.10</td>
|
||
* Reading the pair structurally is what stops the label/value rebinding that
|
||
* flattening causes (I9) — flattened, "Discounts -$24.09 Total Charged $14.64"
|
||
* invites a regex to bind the wrong number to the wrong label.
|
||
*/
|
||
function tdPairValue(html: string, label: string): number | null {
|
||
const re = new RegExp(
|
||
`<td[^>]*>\\s*${label}\\s*</td>\\s*<td[^>]*>\\s*(-?\\s*\\$?[\\d,]+\\.\\d{2})\\s*</td>`,
|
||
"i"
|
||
);
|
||
const m = html.match(re);
|
||
return m ? money(m[1]) : null;
|
||
}
|
||
|
||
/**
|
||
* Uber itemises only *grocery* orders — a restaurant receipt states a total and
|
||
* nothing else, which is why 67 of the 101 backfilled orders have no items.
|
||
* When it does itemise, the markup is far better than DoorDash's: every cell
|
||
* carries a `data-testid` naming its role and the item's own uuid, so quantity,
|
||
* title and amount can be bound to each other by id rather than by position.
|
||
*/
|
||
function parseUberLineItems(html: string): LineItem[] {
|
||
const items: LineItem[] = [];
|
||
const titleRe =
|
||
/data-testid="shoppingCart_item_title_([0-9a-f-]+)"[^>]*>([\s\S]*?)<\/td>/gi;
|
||
|
||
for (const m of html.matchAll(titleRe)) {
|
||
const [, id, rawTitle] = m;
|
||
const description = collapse(decodeEntities(stripTags(rawTitle)));
|
||
if (!description) continue;
|
||
|
||
const qtyM = html.match(
|
||
new RegExp(`data-testid="shoppingCart_item_quantity_${id}"[^>]*>\\s*(\\d+)\\s*<`, "i")
|
||
);
|
||
const amtM = html.match(
|
||
new RegExp(
|
||
`data-testid="shoppingCart_item_amount_${id}"[^>]*>([\\s\\S]*?)<\\/td>`,
|
||
"i"
|
||
)
|
||
);
|
||
const amtText = amtM ? collapse(decodeEntities(stripTags(amtM[1]))) : "";
|
||
const amtNum = amtText.match(/(-?[\d,]+\.\d{2})/);
|
||
|
||
items.push({
|
||
qty: qtyM ? parseInt(qtyM[1], 10) : 1,
|
||
description,
|
||
// A sold-out item prints 0.00 and is genuinely part of the order — it
|
||
// explains a total that does not match what was asked for. Keep it.
|
||
amount: amtNum ? money(amtNum[1]) : 0,
|
||
});
|
||
}
|
||
return items;
|
||
}
|
||
|
||
/**
|
||
* Uber's `Order details` block, anchored on `data-testid="address_point_N_*"`.
|
||
*
|
||
* The template repeats the whole block twice (once hidden for narrow screens),
|
||
* so the same stop appears more than once and has to be de-duplicated. This is
|
||
* the same markup a *trip* receipt uses for its start and destination — rides
|
||
* are not ingested today, but the reader will not need changing when they are.
|
||
*/
|
||
function parseUberRoute(html: string): RoutePoint[] {
|
||
const seen = new Set<string>();
|
||
const points: RoutePoint[] = [];
|
||
|
||
const labelRe = /data-testid="address_point_(\d+)_time"[^>]*>([\s\S]*?)<\/td>/gi;
|
||
for (const m of html.matchAll(labelRe)) {
|
||
const [, idx, rawLabel] = m;
|
||
const addrM = html.match(
|
||
new RegExp(`data-testid="address_point_${idx}_address"[^>]*>([\\s\\S]*?)<\\/td>`, "i")
|
||
);
|
||
if (!addrM) continue;
|
||
|
||
const address = collapse(decodeEntities(stripTags(addrM[1])));
|
||
// Delivery receipts share one cell between time and label — "1:20 pm -
|
||
// Pick-up". Trip receipts print the time alone, with no label at all, so
|
||
// the naive split put the time in `label` and left `time` null. Position
|
||
// carries the meaning there: first stop is where the ride began.
|
||
const combined = collapse(decodeEntities(stripTags(rawLabel)));
|
||
const split = combined.match(/^(.*?)\s+-\s+(.*)$/);
|
||
let time: string | null;
|
||
let label: string;
|
||
if (split) {
|
||
time = split[1];
|
||
label = split[2];
|
||
} else if (/^\d{1,2}:\d{2}\s*(am|pm)?$/i.test(combined)) {
|
||
time = combined;
|
||
label = ""; // filled in positionally below — the receipt gives none
|
||
} else {
|
||
time = null;
|
||
label = combined;
|
||
}
|
||
|
||
const key = `${label}|${time}|${address}`;
|
||
if (!address || seen.has(key)) continue;
|
||
seen.add(key);
|
||
points.push({ label, time, address });
|
||
}
|
||
|
||
// A trip receipt labels neither end. Position is the only thing that says
|
||
// which is which, and for a two-stop trip it says it unambiguously. Only
|
||
// filled where the receipt itself was silent, so a future template that does
|
||
// label its stops keeps its own wording.
|
||
if (points.length === 2 && points.every((p) => !p.label)) {
|
||
points[0].label = "Pick-up";
|
||
points[1].label = "Drop-off";
|
||
}
|
||
return points;
|
||
}
|
||
|
||
function parseDoorDashLineItems(html: string): LineItem[] {
|
||
// <td width="10%">1x</td><td width="75%"><b>Name</b> (Cat)<br><font>• Opt</font>…</td><td width="15%">$22.10</td>
|
||
const re =
|
||
/<td[^>]*width="10%"[^>]*>\s*(\d+)x\s*<\/td>\s*<td[^>]*width="75%"[^>]*>([\s\S]*?)<\/td>\s*<td[^>]*width="15%"[^>]*>\s*\$?([\d,]+\.\d{2})\s*<\/td>/gi;
|
||
const items: LineItem[] = [];
|
||
for (const m of html.matchAll(re)) {
|
||
const parts = decodeEntities(stripTags(m[2]))
|
||
.split("•")
|
||
.map((p) => collapse(p))
|
||
.filter(Boolean);
|
||
if (!parts.length) continue;
|
||
items.push({
|
||
qty: parseInt(m[1], 10),
|
||
description: parts[0],
|
||
amount: money(m[3]),
|
||
options: parts.slice(1),
|
||
});
|
||
}
|
||
return items;
|
||
}
|
||
|
||
/** Reads "<Label> [CUR] 1,234.56" out of already-flattened Uber receipt text. */
|
||
function extractLabelled(text: string, label: string): number | null {
|
||
const m = text.match(
|
||
new RegExp(`${label}\\s*(?:[A-Z]{3})?\\s*\\$?\\s*([\\d,]+\\.\\d{2})`, "i")
|
||
);
|
||
return m ? money(m[1]) : null;
|
||
}
|
||
|
||
function detectPlatform(meta: MessageMeta, html: string): ParsedOrder["platform"] {
|
||
const s = meta.subject || "";
|
||
const from = (meta.sender || "").toLowerCase();
|
||
if (/doordash/i.test(from) || /^\s*(\[Family\]\s*)?Order Confirmation for/i.test(s)) {
|
||
return "doordash";
|
||
}
|
||
if (/order with Uber Eats/i.test(s)) return "ubereats";
|
||
if (/trip with Uber|Uber receipt|Trip fare/i.test(s) || /Trip fare/i.test(html)) return "uber";
|
||
throw new NotAReceiptError(`cannot determine platform from subject: ${s}`, meta.messageId);
|
||
}
|
||
|
||
function parseMerchant(platform: string, meta: MessageMeta, text: string): string {
|
||
if (platform === "doordash") {
|
||
// Subject: "Order Confirmation for Siddharth from Mad Mex"
|
||
const m = meta.subject.match(/Order Confirmation for \S+\s+from\s+([\s\S]+?)\s*$/i);
|
||
if (m) return collapse(m[1]);
|
||
}
|
||
if (platform === "ubereats") {
|
||
// Body: "Here's your receipt for TEG Kebabs & Biryani."
|
||
const m = text.match(/receipt for\s+([\s\S]+?)\s*\.\s/i);
|
||
if (m) return collapse(m[1]);
|
||
}
|
||
if (platform === "uber") return "Uber Trip";
|
||
throw new OrderParseError(`cannot determine merchant`, meta.messageId);
|
||
}
|
||
|
||
/**
|
||
* Currency notations Uber actually uses in receipt totals.
|
||
*
|
||
* Deliberately narrow: only prefixes that are unambiguous. "R$" (BRL) and "$"
|
||
* alone are excluded — a bare dollar sign is used by a dozen currencies and
|
||
* resolving it here would overrule the body-wide scan that reads the receipt's
|
||
* own stated code.
|
||
*/
|
||
const SYMBOL_PREFIX_CURRENCY: Record<string, string> = {
|
||
A: "AUD",
|
||
NZ: "NZD",
|
||
US: "USD",
|
||
S: "SGD",
|
||
HK: "HKD",
|
||
C: "CAD",
|
||
};
|
||
|
||
const SYMBOL_CURRENCY: Record<string, string> = {
|
||
"₹": "INR",
|
||
"€": "EUR",
|
||
"£": "GBP",
|
||
};
|
||
|
||
function parsePayment(
|
||
platform: string,
|
||
html: string,
|
||
text: string,
|
||
statedTotal: number | null = null
|
||
): PaymentBreakdown {
|
||
const out: PaymentBreakdown = {
|
||
credits_amount: null,
|
||
card_amount: null,
|
||
card_last4: null,
|
||
ambiguous: false,
|
||
};
|
||
|
||
if (platform === "doordash") {
|
||
// Match the instrument directly. An earlier version captured a trailing
|
||
// window delimited by a double space, which does not survive whitespace
|
||
// collapsing — so every card/mixed receipt fell through to the credits
|
||
// branch and booked the full total as credits.
|
||
if (/Paid with[\s\S]{0,60}?and\/or\s*credits/i.test(text)) {
|
||
out.ambiguous = true;
|
||
const l4 = text.match(/Paid with[\s\S]{0,60}?Ending in\s*(\d{3,4})/i);
|
||
out.card_last4 = l4 ? l4[1] : null;
|
||
return out;
|
||
}
|
||
const card = text.match(/Paid with[\s\S]{0,40}?Ending in\s*(\d{3,4})/i);
|
||
if (card) {
|
||
out.card_last4 = card[1];
|
||
return out; // card-only; amount filled from total
|
||
}
|
||
if (/Paid with\s+credits/i.test(text)) return out; // credits-only
|
||
return out;
|
||
}
|
||
|
||
// Uber Eats / Uber: "Payments Uber Cash $25.33", or a card line, or — on
|
||
// [Family] orders placed on a shared account — just the payer's name:
|
||
// "Payments Siddharth LKR 3,783.20". That last form names no instrument at
|
||
// all, so the split cannot be read from the mail and must not be invented.
|
||
// A declined attempt is still printed, immediately followed by "Failed":
|
||
// Visa ••••8841 LKR 4,267.01 7/5/26 7:20 pm Failed
|
||
// Siddharth LKR 3,757.01 7/5/26 9:02 pm
|
||
// (grocery order re-charged lower after sold-out items). Taking the first
|
||
// match would record money that never left the account, so drop the failed
|
||
// attempts before reading any instrument.
|
||
text = text.replace(
|
||
/(?:Visa|MasterCard|Amex|American Express|Uber Cash)[^.]{0,40}?[\d,]+\.\d{2}\s+\S+\s+\S+\s*(?:am|pm)?\s*Failed/gi,
|
||
" "
|
||
);
|
||
|
||
// "Payments Uber Cash $25.33" — but newer receipts put a timestamp between
|
||
// the label and the amount, and write the currency as a prefix:
|
||
// "Payments Uber Cash 10/17/25 8:50 PM A$54.87"
|
||
// Both defeated the old pattern, and the failure was silent and expensive:
|
||
// Uber Cash IS credits, so an unreadable payment line left credits_amount
|
||
// null and the order was filed as card-settled. It then went looking for a
|
||
// card leg that does not exist, found nothing, and became an orphan with no
|
||
// transaction and no card to match on. 118 of the captured messages sit in
|
||
// that state. The date is allowed for explicitly rather than by widening the
|
||
// gap, so a distant unrelated amount still cannot be captured.
|
||
const cash = text.match(
|
||
/Uber Cash\s*(?:\d{1,2}\/\d{1,2}\/\d{2,4}\s*)?(?:\d{1,2}:\d{2}\s*(?:AM|PM)?\s*)?(?:([A-Z]{3})\s*)?(?:[A-Z]{1,2})?[$₹€£]?\s*([\d,]+\.\d{2})/i
|
||
);
|
||
if (cash) out.credits_amount = money(cash[2]);
|
||
// Anchor on the masking, not on a list of card brands. Uber labels the card
|
||
// leg with whatever the issuer is called — "Westpac ••••8032 $15.33",
|
||
// "Mastercard ••••3893 (CBA Ultimate) CHF 51.23" — so a brand allowlist
|
||
// silently drops the card half of a mixed payment. Found in the backfill
|
||
// dry-run: Uber Cash $1.17 + Westpac ••••8032 $15.33 against a $16.50 total,
|
||
// which validateOrderTotals correctly refused rather than under-recording.
|
||
// The gap was `[^\d]{0,40}` — no digits — which the newer layout breaks by
|
||
// printing a timestamp there: "Westpac ••••8032 2/14/25 1:59 PM A$8.68".
|
||
// That silently dropped the card half of every mixed payment in the new
|
||
// format, and stayed invisible only while Uber Cash was also unreadable:
|
||
// both legs missing meant the order looked card-settled and the whole total
|
||
// was booked to a card. Reading credits without fixing this reads half an
|
||
// order, and validateOrderTotals rightly refuses it.
|
||
//
|
||
// Every leg is summed rather than just the first. One order can be charged
|
||
// in several instalments to the SAME card — a Dubai trip billed as
|
||
// "Citi Prestige ••••0253 7/2/25 AED 17.67" and again the next day, totalling
|
||
// 577.83 — and an instrument can carry no mask at all ("PayPal - <email>").
|
||
// Taking one match under-reads both, and the order is then refused for a
|
||
// shortfall the receipt does not actually have.
|
||
const legs = [
|
||
...text.matchAll(
|
||
/(?:(?:••••|\*{4}|\u2022{4})\s*(\d{4})|PayPal)[^$₹€£]{0,60}?(?:([A-Z]{3})\s+|(?:[A-Z]{1,2})?[$₹€£]\s?)([\d,]+\.\d{2})/g
|
||
),
|
||
];
|
||
//
|
||
// Summing is right for instalments but wrong for a re-auth. A Dubai trip
|
||
// prints "Citi Prestige ••••0253 AED 17.67" (the authorisation) and then
|
||
// "Citi Prestige ••••0253 AED 577.83" (the settled charge) against a stated
|
||
// total of 577.83 — the first leg is superseded, not additive, and adding it
|
||
// overstates the trip by the held amount. So a leg that already equals the
|
||
// stated total IS the payment; only when none does are the legs instalments
|
||
// that must be added. A mixed credits+card order is unaffected: neither leg
|
||
// equals the total there, which is exactly why it needs summing.
|
||
const exact = legs.find(
|
||
(l) => statedTotal !== null && Math.abs(money(l[3]) - statedTotal) < 0.02
|
||
);
|
||
if (exact && out.credits_amount === null) {
|
||
out.card_last4 = exact[1] ?? null;
|
||
out.card_amount = money(exact[3]);
|
||
} else {
|
||
let cardTotal = 0;
|
||
for (const leg of legs) {
|
||
cardTotal += money(leg[3]);
|
||
if (leg[1] && !out.card_last4) out.card_last4 = leg[1];
|
||
}
|
||
if (legs.length > 0) out.card_amount = Number(cardTotal.toFixed(2));
|
||
}
|
||
if (!cash && legs.length === 0 && /Payments\s+\S+\s+(?:[A-Z]{3}\s|\$)/.test(text)) {
|
||
out.ambiguous = true;
|
||
}
|
||
return out;
|
||
}
|
||
|
||
export function parseOrderHTML(html: string, meta: MessageMeta): ParsedOrder {
|
||
if (!html || html.length < 200) {
|
||
throw new NotAReceiptError("body too short to be a receipt", meta.messageId);
|
||
}
|
||
const clean = html.replace(/<!--[\s\S]*?-->/g, "");
|
||
const text = collapse(decodeEntities(stripTags(clean)));
|
||
const flags: string[] = [];
|
||
let explicitCurrency: string | null = null;
|
||
|
||
// Grocery orders (ALDI, Woolworths) generate a follow-up "There are
|
||
// adjustments to your order" mail for out-of-stock and substituted items. It
|
||
// reuses the receipt layout but states `Total: $0.00` — the real amount is
|
||
// settled later. Ingesting it would book a $0.00 order and, worse, its
|
||
// order_reference would collide with nothing and create a phantom row.
|
||
// Refund / total-adjustment mails ("We adjusted the total for your recent
|
||
// order", "Previous total ... Refund ... New Total"). These restate an order
|
||
// already ingested and must be applied as an amendment, not inserted as a new
|
||
// order. Amendment handling is not in this pass — reject loudly so none is
|
||
// silently double-counted.
|
||
if (/We adjusted the total|Your refund has been applied|Previous total/i.test(text)) {
|
||
throw new NotAReceiptError(
|
||
"refund/total-adjustment notice — amends an existing order, not a new receipt",
|
||
meta.messageId
|
||
);
|
||
}
|
||
|
||
if (/There are adjustments to your order/i.test(text)) {
|
||
throw new NotAReceiptError(
|
||
"order-adjustment notice, not a receipt — no final total stated",
|
||
meta.messageId
|
||
);
|
||
}
|
||
|
||
// Uber sends TWO mails per trip with the same subject and the same total: a
|
||
// "charge summary" when the trip ends, then the real receipt once payment
|
||
// settles. The summary says so itself — "This is not a payment receipt ...
|
||
// You will receive a trip receipt when the payment is processed with payment
|
||
// information" — and it carries no tripReference, so order_reference would
|
||
// fall back to `msg:<message-id>` and I7 could not dedupe it against the
|
||
// receipt that follows. Every trip would be recorded twice.
|
||
if (/This is not a payment receipt|This is your charge summary/i.test(text)) {
|
||
throw new NotAReceiptError(
|
||
"charge summary, not a payment receipt — the real receipt follows",
|
||
meta.messageId
|
||
);
|
||
}
|
||
|
||
const platform = detectPlatform(meta, text);
|
||
const merchant_name = parseMerchant(platform, meta, text);
|
||
|
||
// ---- order_reference -----------------------------------------------------
|
||
// Uber embeds a real order UUID in the body. DoorDash embeds no order id at
|
||
// all, so the provider message id is the only stable identity available —
|
||
// which is correct for ingestion idempotency (one receipt = one order).
|
||
//
|
||
// Anchor on Uber's own `tripReference` cell — a hidden
|
||
// <td class="tripReference">xid<UUID></td> present in all 29 captured
|
||
// receipts. For ue-00 it equals the UUID the PDF redirect resolves to
|
||
// (ubereats.com/orders/34d6b4ee-...), so it is the order's real identity.
|
||
//
|
||
// The alternative, "first UUID in the document", is positional rather than
|
||
// semantic: 4 of 29 receipts carry several UUIDs, and if a template reshuffle
|
||
// ever put a per-send tracking id first, the symptom would be a reference
|
||
// that changes every fetch and silently duplicates every order on every
|
||
// backfill. Fall back to it only when the anchor is absent, and flag when
|
||
// that fallback is genuinely ambiguous.
|
||
let order_reference: string;
|
||
const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;
|
||
const anchored = clean.match(
|
||
/tripReference[^>]*>\s*xid([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i
|
||
);
|
||
const firstUuid = clean.match(UUID_RE);
|
||
if (platform !== "doordash" && (anchored || firstUuid)) {
|
||
order_reference = (anchored ? anchored[1] : firstUuid![0]).toLowerCase();
|
||
if (!anchored) {
|
||
const distinct = new Set(
|
||
(clean.match(new RegExp(UUID_RE.source, "gi")) || []).map((u) => u.toLowerCase())
|
||
);
|
||
// Only ambiguous when there is more than one candidate to choose between.
|
||
if (distinct.size > 1) flags.push("order_uuid_ambiguous");
|
||
}
|
||
} else {
|
||
// DoorDash carries no order id anywhere in the receipt, so the provider
|
||
// message id is the only stable identity available. That is correct for
|
||
// ingestion idempotency: one receipt is one order.
|
||
if (!meta.messageId) {
|
||
throw new OrderParseError("no order id in body and no messageId supplied");
|
||
}
|
||
order_reference = `msg:${meta.messageId}`;
|
||
}
|
||
|
||
// ---- date ----------------------------------------------------------------
|
||
// From the message, never the body. DoorDash confirmations are sent at order
|
||
// time; the body's own date strings are inconsistent and locale-formatted.
|
||
const received = new Date(meta.receivedAt);
|
||
if (isNaN(received.getTime())) {
|
||
throw new OrderParseError(`unparseable receivedAt: ${meta.receivedAt}`, meta.messageId);
|
||
}
|
||
const order_datetime = received.toISOString();
|
||
|
||
// ---- [Family] ------------------------------------------------------------
|
||
// A subject prefix on Uber Eats: "[Family] Your Sunday evening order with…".
|
||
// Deliberately anchored — a bare substring search for "family" matches
|
||
// footer copy and merchant names, and a false positive here silently drops
|
||
// the order out of every budget.
|
||
const is_family = /^\s*\[Family\]/i.test(meta.subject || "");
|
||
|
||
// ---- totals --------------------------------------------------------------
|
||
let totals: OrderTotals;
|
||
if (platform === "doordash") {
|
||
// Grocery "Final receipt" mails price each item and carry no Total Charged
|
||
// row; the only stated total is the header. Fall back to it explicitly
|
||
// rather than letting a partial total through.
|
||
const headerTotal = text.match(/Total:\s*\$?([\d,]+\.\d{2})/i);
|
||
const total =
|
||
tdPairValue(clean, "Total Charged") ??
|
||
tdPairValue(clean, "Total") ??
|
||
(headerTotal ? money(headerTotal[1]) : null);
|
||
if (total === null) {
|
||
throw new OrderParseError("no total stated anywhere in receipt", meta.messageId);
|
||
}
|
||
const subtotal = tdPairValue(clean, "Subtotal");
|
||
totals = {
|
||
subtotal,
|
||
taxes: tdPairValue(clean, "Taxes"),
|
||
delivery_fee: tdPairValue(clean, "Delivery Fee"),
|
||
service_fee: tdPairValue(clean, "Service Fee"),
|
||
tip: tdPairValue(clean, "Tip"),
|
||
discounts: tdPairValue(clean, "Discounts"),
|
||
total_charged: total,
|
||
};
|
||
|
||
// An order paid entirely from DoorDash credits states "Total Charged
|
||
// $0.00" — truthfully, because nothing was charged to a card — while the
|
||
// items above it add up to a real amount. Read literally that is a $0
|
||
// order, and `validateOrderTotals` rejected 88 of them as "non-positive
|
||
// total 0", which is the single largest cause of parse failures in the
|
||
// captured mail and discards exactly the credit-funded spend this pipeline
|
||
// exists to make visible.
|
||
//
|
||
// The order's value is its subtotal. Recording that keeps a credits meal
|
||
// countable in budgets; recording zero would show the order and hide what
|
||
// it cost. Guarded on the receipt actually saying credits, so a genuinely
|
||
// zero-value mail still fails rather than inheriting a stray subtotal.
|
||
if (total === 0 && subtotal !== null && subtotal > 0 && /Paid with[\s\S]{0,60}?credits/i.test(text)) {
|
||
totals.total_charged = subtotal;
|
||
flags.push("credits_funded_zero_charge");
|
||
}
|
||
} else {
|
||
// Uber Eats states a Total, optionally in a foreign currency. Uber writes
|
||
// the currency in three different notations and all three occur in real
|
||
// mail:
|
||
// "Total $25.33" bare — the home currency
|
||
// "Total LKR 3,783.20" ISO code, space-separated
|
||
// "Total A$54.87" symbol-prefixed: A$, NZ$, US$, S$, HK$, C$
|
||
// "Total ₹1,240.00" a bare symbol
|
||
// Only the first two were handled. The prefixed form is not exotic: it is
|
||
// what Uber sends for ordinary Australian orders, so 176 of 550 captured
|
||
// messages — most of them 2024-2025, i.e. current mail rather than legacy
|
||
// templates — failed with "no Total found" while the amount sat in plain
|
||
// sight in the body. `A$` misses `[A-Z]{3}` by a character.
|
||
//
|
||
// The [Family] orders are placed for family in Sri Lanka and are priced in
|
||
// LKR — reading those as dollars would inflate them ~200x, which is a large
|
||
// part of why they must not reach a budget untagged.
|
||
const m = text.match(
|
||
/(?:New Total|Total)\s*(?:([A-Z]{3})\s*)?(?:([A-Z]{1,2})?\$|([₹€£]))?\s*([\d,]+\.\d{2})/
|
||
);
|
||
if (!m) throw new OrderParseError("no Total found", meta.messageId);
|
||
totals = {
|
||
subtotal: extractLabelled(text, "Item subtotal"),
|
||
taxes: extractLabelled(text, "Tax"),
|
||
delivery_fee: extractLabelled(text, "Delivery Fee"),
|
||
service_fee: extractLabelled(text, "Service Fee"),
|
||
tip: null,
|
||
discounts: null,
|
||
total_charged: money(m[4]),
|
||
};
|
||
// A symbol is only evidence of currency when it is qualified. A bare "$"
|
||
// stays unset so the body-wide scan below still gets its say — the receipt
|
||
// often names the currency elsewhere, and guessing AUD here would overrule
|
||
// it.
|
||
explicitCurrency =
|
||
(m[1] && m[1].toUpperCase()) ||
|
||
(m[2] && SYMBOL_PREFIX_CURRENCY[m[2].toUpperCase()]) ||
|
||
(m[3] && SYMBOL_CURRENCY[m[3]]) ||
|
||
explicitCurrency;
|
||
}
|
||
|
||
// ---- payment -------------------------------------------------------------
|
||
const payment = parsePayment(platform, clean, text, totals.total_charged);
|
||
if (payment.ambiguous && is_family) {
|
||
// [Family] receipts name the payer, not an instrument ("Payments Siddharth
|
||
// LKR 3,783.20"). An earlier version read that as credits-funded. It is
|
||
// not: the card statement carries all four of them (CBA ...3893, exact
|
||
// foreign_currency_amount matches), so creating a transaction duplicated
|
||
// spend that was already recorded — precisely the double-count I5 exists to
|
||
// prevent.
|
||
//
|
||
// Treated as card-settled: provenance only, no transaction. The statement
|
||
// line IS the transaction, and it is what should carry the `family` tag.
|
||
flags.push("family_card_settled_no_transaction");
|
||
} else if (payment.ambiguous) {
|
||
// Split not stated and resolvable from the card statement — left for the
|
||
// ingestion runner to reconcile, not guessed here.
|
||
flags.push("payment_split_not_stated");
|
||
} else if (platform === "doordash") {
|
||
// DoorDash names the method but not the amount; the total is the amount.
|
||
if (payment.card_last4) payment.card_amount = totals.total_charged;
|
||
else payment.credits_amount = totals.total_charged;
|
||
}
|
||
|
||
// ---- line items ----------------------------------------------------------
|
||
// Uber Eats receipts carry no itemisation (verified across 29 real mails).
|
||
const line_items =
|
||
platform === "doordash" ? parseDoorDashLineItems(clean) : parseUberLineItems(clean);
|
||
if (platform === "doordash" && line_items.length === 0) {
|
||
flags.push("no_line_items_parsed");
|
||
}
|
||
|
||
// Uber prints addresses on every receipt; DoorDash prints none at all, so an
|
||
// empty route there is expected rather than a parse failure.
|
||
const route = platform === "doordash" ? [] : parseUberRoute(clean);
|
||
|
||
const currency =
|
||
explicitCurrency ||
|
||
(/\b(NZD|USD|LKR|CHF|EUR|GBP|SGD|INR)\b/.test(text)
|
||
? (text.match(/\b(NZD|USD|LKR|CHF|EUR|GBP|SGD|INR)\b/) as RegExpMatchArray)[1]
|
||
: "AUD");
|
||
|
||
return {
|
||
order_reference,
|
||
platform,
|
||
merchant_name,
|
||
order_datetime,
|
||
currency,
|
||
payment,
|
||
totals,
|
||
line_items,
|
||
route,
|
||
is_family,
|
||
flags,
|
||
};
|
||
}
|
||
|
||
/**
|
||
* Integrity check on the amount we are about to record as spend.
|
||
*
|
||
* NOT an arithmetic reconciliation of the fee breakdown. Measured against 36
|
||
* real DoorDash receipts, the components do not sum to the total on 32 of them:
|
||
* DoorDash's `Discounts` line frequently equals subtotal + service fee exactly
|
||
* (Mad Mex: subtotal 22.10, service 1.99, Discounts -24.09, Total Charged
|
||
* 14.64) and sometimes differs by an unrelated margin. Whatever that line means
|
||
* to DoorDash, it is not a term in `total = components`.
|
||
*
|
||
* This corrects an earlier diagnosis that read the same numbers as an
|
||
* HTML-flattening artefact with a "true discount of $9.45". Parsing the table
|
||
* cells structurally yields the identical figures, and no $9.45 appears
|
||
* anywhere in the message — the receipt genuinely says this.
|
||
*
|
||
* So the breakdown is stored as provenance and never gated on. What IS checked
|
||
* is the number that becomes money in the ledger: DoorDash states the total
|
||
* twice, independently (a `Total: $X` header and a `Total Charged` table row),
|
||
* and those must agree. That catches a mis-parse, which is the failure that
|
||
* actually matters.
|
||
*/
|
||
export function validateOrderTotals(
|
||
order: ParsedOrder,
|
||
html?: string
|
||
): { ok: boolean; reason?: string } {
|
||
const t = order.totals;
|
||
|
||
if (!(t.total_charged > 0)) {
|
||
return { ok: false, reason: `non-positive total ${t.total_charged}` };
|
||
}
|
||
|
||
// Cross-check the header total against the table total where both exist.
|
||
//
|
||
// Skipped for a credits-funded order: both stated totals are $0.00 there and
|
||
// agree with each other, but the recorded amount is deliberately the
|
||
// subtotal, so the check would reject every one of them for "disagreeing"
|
||
// with a figure the parser overrode on purpose.
|
||
if (
|
||
html &&
|
||
order.platform === "doordash" &&
|
||
!order.flags.includes("credits_funded_zero_charge")
|
||
) {
|
||
const header = collapse(decodeEntities(stripTags(html))).match(
|
||
/Total:\s*\$?([\d,]+\.\d{2})/i
|
||
);
|
||
if (header) {
|
||
const stated = money(header[1]);
|
||
if (Math.abs(stated - t.total_charged) > 0.02) {
|
||
return {
|
||
ok: false,
|
||
reason: `header total ${stated.toFixed(2)} disagrees with Total Charged ${t.total_charged.toFixed(2)}`,
|
||
};
|
||
}
|
||
}
|
||
}
|
||
|
||
// The payment line must account for the total, or we are recording an amount
|
||
// no stated payment method covers.
|
||
if (!order.payment.ambiguous) {
|
||
const paid = (order.payment.credits_amount || 0) + (order.payment.card_amount || 0);
|
||
if (paid > 0 && Math.abs(paid - t.total_charged) > 0.02) {
|
||
return {
|
||
ok: false,
|
||
reason: `payments sum to ${paid.toFixed(2)} but receipt states ${t.total_charged.toFixed(2)}`,
|
||
};
|
||
}
|
||
}
|
||
|
||
return { ok: true };
|
||
}
|
||
|
||
|
||
export interface OrderAmendment {
|
||
order_reference: string | null;
|
||
previous_total: number | null;
|
||
refund_amount: number | null;
|
||
new_total: number;
|
||
order_datetime: string;
|
||
messageId: string;
|
||
}
|
||
|
||
/**
|
||
* Refund / total-adjustment notices restate an order that was already ingested:
|
||
*
|
||
* "We adjusted the total for your recent order from Coles (Wyndham Vale)."
|
||
* Previous total $49.94 · Refund -$4.21 · New Total $45.73
|
||
*
|
||
* These are amendments, not receipts — inserting one as a new order would
|
||
* double-count the meal and hide the refund. Uber embeds the same order UUID it
|
||
* used on the original receipt, so the amendment can be matched back to it.
|
||
*/
|
||
export function parseOrderAmendment(html: string, meta: MessageMeta): OrderAmendment {
|
||
const clean = html.replace(/<!--[\s\S]*?-->/g, "");
|
||
const text = collapse(decodeEntities(stripTags(clean)));
|
||
|
||
if (!/We adjusted the total|Your refund has been applied|Previous total/i.test(text)) {
|
||
throw new OrderParseError("not an amendment notice", meta.messageId);
|
||
}
|
||
|
||
const newTotal = text.match(/New Total\s*(?:[A-Z]{3})?\s*\$?\s*([\d,]+\.\d{2})/i);
|
||
if (!newTotal) {
|
||
throw new OrderParseError("amendment states no New Total", meta.messageId);
|
||
}
|
||
const prev = text.match(/Previous total\s*(?:[A-Z]{3})?\s*\$?\s*([\d,]+\.\d{2})/i);
|
||
const refund = text.match(/Refund\s*-?\s*(?:[A-Z]{3})?\s*\$?\s*([\d,]+\.\d{2})/i);
|
||
const uuid = clean.match(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i);
|
||
|
||
const received = new Date(meta.receivedAt);
|
||
if (isNaN(received.getTime())) {
|
||
throw new OrderParseError(`unparseable receivedAt: ${meta.receivedAt}`, meta.messageId);
|
||
}
|
||
|
||
return {
|
||
order_reference: uuid ? uuid[0].toLowerCase() : null,
|
||
previous_total: prev ? money(prev[1]) : null,
|
||
refund_amount: refund ? money(refund[1]) : null,
|
||
new_total: money(newTotal[1]),
|
||
order_datetime: received.toISOString(),
|
||
messageId: meta.messageId,
|
||
};
|
||
}
|
||
|
||
/** True when a message is an amendment rather than a receipt. */
|
||
export function isAmendment(html: string): boolean {
|
||
const text = collapse(decodeEntities(stripTags(html.replace(/<!--[\s\S]*?-->/g, ""))));
|
||
return /We adjusted the total|Your refund has been applied|Previous total/i.test(text);
|
||
}
|