feat(orders): wire the real parser in, defer card reconciliation

Ingestion now runs on the rebuilt parser. Three substantive changes.

Deferred card reconciliation. A 'MasterCard 8032 and/or credits' receipt never
states the split, but the card leg lands on the statement — Subway's $29.08
order shows $13.06 on 8032, so $16.02 was credits. For a live order that
statement is weeks away, so the split cannot be settled at ingest time. Such
orders are now parked with provenance and no transaction, and
reconcilePendingOrders() resolves them once the statement arrives. Backfill
takes the same path and resolves immediately. Migration 0019 adds the columns
that make an order resumable; applied to personal_test only, prod untouched.

Payment detection bug, found by the new tests: the old regex delimited the
'Paid with' line on a double space, which whitespace collapsing removes. Every
card and mixed receipt fell through to the credits branch — the Woolworths
receipt booked $60.93 of credits spend that never happened.

Category resolution reversed deliberately. Correction 1 said never default to
dining; the implementation of that sent everything unrecognised to 'other', and
knowing six merchants meant Carl's Jr, Taco Bell, Chilli India, Oporto, Schnitz
and Souvlaki GR all landed there. Grocers are an enumerable set and restaurants
are not, so match groceries explicitly and let the residual be dining.

Tests rebuilt on real captured receipts; the synthetic fixtures are deleted.
60 unit + 41 integration green on three consecutive runs.
This commit is contained in:
2026-07-26 22:43:10 +10:00
parent 33db7d05ef
commit c82a22767f
10 changed files with 626 additions and 531 deletions
+259 -235
View File
@@ -1,273 +1,184 @@
import { queryRaw, queryRow } from "./db";
import { EXCLUDE_NON_SPEND } from "./analytics-sql";
import type { ParsedOrder } from "./order-parse";
export interface LineItem {
qty: number;
description: string;
amount: number;
options?: string[];
}
export * from "./order-parse";
export interface PaymentBreakdown {
credits_amount: number | null;
card_amount: number | null;
card_last4: string | null;
}
export const CUTOVER_DATE = "2026-01-09";
export interface OrderTotals {
subtotal: number;
taxes: number | null;
delivery_fee: number | null;
service_fee: number | null;
tip: number | null;
discounts: number | null;
total_charged: number;
}
export interface ParsedOrder {
order_reference: string;
platform: "doordash" | "ubereats" | "uber";
merchant_name: string;
order_datetime: string;
currency: string;
payment: PaymentBreakdown;
totals: OrderTotals;
line_items: LineItem[];
is_family?: boolean;
export interface IngestResult {
transactionId: number | null;
metadataId: number | null;
flags: string[];
skipped?: string;
}
/**
* Extracts structured data from raw receipt HTML.
* Preserves HTML table structure to avoid cell-rebinding bugs (I9).
* Resolves how much of a card-settled order actually hit the card.
*
* DoorDash writes "MasterCard Ending in 8032 and/or credits" without ever
* stating the split. The split is not in the mail — but it IS in the ledger:
* the card leg arrives on the statement for that card. Reconciling against it
* beats guessing (user, 2026-07-26).
*
* Subway receipt $29.08, statement 8032 charge $13.06 -> $16.02 was credits
* lahori receipt $50.85, statement 8032 charge $50.85 -> fully card
*
* Returns the card amount if a statement line can be matched, else null. A null
* means "unknown", and the caller must not invent a credits figure from it.
*/
export function parseOrderHTML(html: string): ParsedOrder {
// Detect platform
const isUberEats = /Uber Eats/i.test(html);
const isUberRide = /UberX|UberXL|Uber Comfort|trip with Uber|Trip fare/i.test(html) || (/Uber/i.test(html) && !isUberEats && !/doordash/i.test(html));
const platform: "doordash" | "ubereats" | "uber" = isUberEats ? "ubereats" : isUberRide ? "uber" : "doordash";
export async function reconcileCardLeg(
order: ParsedOrder,
windowDays = 4
): Promise<{ cardAmount: number | null; matchedTransactionId: number | null }> {
const last4 = order.payment.card_last4;
if (!last4) return { cardAmount: null, matchedTransactionId: null };
// Extract payment line (supports DoorDash "Paid with credits" and Uber "Uber Cash $XX.XX")
const uberCashMatch = html.match(/Uber Cash\s+\$?(\d+\.\d{2})/i);
const creditsMatch = html.match(/Paid with credits(?:\s+\$(\d+\.\d{2}))?/i) || uberCashMatch;
const cardMatch = html.match(/Paid with (?:MasterCard|Visa|American Express|Card|Mastercard)(?:\s+Ending in (\d+)|\s+••••(\d+))?(?:\s+\$?(\d+\.\d{2}))?/i);
const mixedMatch = html.match(/Paid with credits\s+\$(\d+\.\d{2}),?\s+.*Ending in (\d+)\s+\$(\d+\.\d{2})/i);
const day = order.order_datetime.slice(0, 10);
const row = await queryRow<{ id: number; amount: string }>(
`SELECT t.id, t.amount::text
FROM transactions t
JOIN statements s ON s.id = t.statement_id
WHERE replace(s.account_number, '-', '') LIKE $1
AND t.transaction_date BETWEEN $2::date - $4::int AND $2::date + $4::int
AND (t.description ILIKE '%doordash%' OR t.description ILIKE '%uber%')
AND t.amount <= $3::numeric + 0.02
ORDER BY abs(t.amount - $3::numeric), abs(t.transaction_date - $2::date)
LIMIT 1`,
[`%${last4}`, day, order.totals.total_charged, windowDays]
);
let credits_amount: number | null = null;
let card_amount: number | null = null;
let card_last4: string | null = null;
if (!row) return { cardAmount: null, matchedTransactionId: null };
return { cardAmount: Number(row.amount), matchedTransactionId: row.id };
}
if (mixedMatch) {
credits_amount = parseFloat(mixedMatch[1]);
card_last4 = mixedMatch[2];
card_amount = parseFloat(mixedMatch[3]);
} else if (uberCashMatch) {
credits_amount = parseFloat(uberCashMatch[1]);
} else if (creditsMatch) {
credits_amount = creditsMatch[1] ? parseFloat(creditsMatch[1]) : null;
} else if (cardMatch) {
card_last4 = cardMatch[1] || cardMatch[2] || null;
card_amount = cardMatch[3] ? parseFloat(cardMatch[3]) : null;
}
// Parse HTML tables for totals to avoid flattening rebinding (I9)
const extractTableValue = (labelPattern: RegExp): number | null => {
const tableMatch = html.match(new RegExp(`<tr[^>]*>\\s*<td[^>]*>\\s*${labelPattern.source}\\s*</td>\\s*<td[^>]*>\\s*(-?\\$?(\\d+\\.\\d{2}))\\s*</td>\\s*</tr>`, 'i'));
if (tableMatch && tableMatch[2]) {
return Math.abs(parseFloat(tableMatch[2]));
}
const divMatch = html.match(new RegExp(`(?:^|>|\\s)${labelPattern.source}\\b[:\\s]*(-?\\$?(\\d+\\.\\d{2}))`, 'i'));
if (divMatch && divMatch[2]) {
return Math.abs(parseFloat(divMatch[2]));
}
return null;
};
const subtotal = extractTableValue(/(?:Subtotal|Trip fare)/i) ?? 0;
const taxes = extractTableValue(/(?:Taxes|Government Levy)/i);
const delivery_fee = extractTableValue(/Delivery Fee/i);
const service_fee = extractTableValue(/(?:Service Fee|Booking Fee)/i);
const tip = extractTableValue(/Tip/i);
const discounts = extractTableValue(/(?:Discounts|Uber One Credits)/i);
const total_charged = extractTableValue(/Total Charged/i) ?? extractTableValue(/Total/i) ?? subtotal;
// If credits/card amounts were not explicit on single payment method, set total_charged
if (creditsMatch && !mixedMatch && credits_amount === null) {
credits_amount = total_charged;
}
if (cardMatch && !mixedMatch && card_amount === null) {
card_amount = total_charged;
}
// Parse merchant
let merchant_name = "Uber";
if (platform === "uber") {
merchant_name = "Uber Trip";
} else {
const merchantMatch = html.match(/<div>\s*([A-Za-z0-9\s'&-]+)\s*<\/div>\s*<div>\s*Total:/i) ||
html.match(/order with\s+([A-Za-z0-9\s'&-]+)/i) ||
html.match(/<div>\s*([A-Za-z0-9\s'&-]+)\s*<\/div>/i);
merchant_name = merchantMatch ? merchantMatch[1].trim() : "Uber Eats Merchant";
}
// Parse line items (Uber Eats has line_items: [] per Difference 1; Uber Rides parse fare components per Difference 2)
const line_items: LineItem[] = [];
if (platform === "uber") {
// Uber Ride fare components
const tripFare = extractTableValue(/Trip fare/i);
if (tripFare) line_items.push({ qty: 1, description: "Trip fare", amount: tripFare });
const bookingFee = extractTableValue(/Booking Fee/i);
if (bookingFee) line_items.push({ qty: 1, description: "Booking Fee", amount: bookingFee });
const airportFee = extractTableValue(/Airport fee/i);
if (airportFee) line_items.push({ qty: 1, description: "Airport fee", amount: airportFee });
} else if (platform === "doordash") {
const lineItemMatch = html.match(/(\d+)x\s+([^$]+)\s+\$(\d+\.\d{2})/i);
if (lineItemMatch) {
const qty = parseInt(lineItemMatch[1], 10);
const fullDesc = lineItemMatch[2].trim();
const parts = fullDesc.split('•').map(p => p.trim());
const description = parts[0];
const options = parts.slice(1);
const amount = parseFloat(lineItemMatch[3]);
line_items.push({ qty, description, amount, options });
}
}
// Currency extraction
const currencyMatch = html.match(/\b(AUD|NZD|LKR|USD)\b/i);
const currency = currencyMatch ? currencyMatch[1].toUpperCase() : "AUD";
// Date parsing
const dateMatch = html.match(/Date:\s*(\d{4}-\d{2}-\d{2})/i);
const order_datetime = dateMatch ? `${dateMatch[1]}T12:00:00Z` : "2026-01-15T12:00:00Z";
// Check if family order (Difference 4)
const is_family = html.includes("[Family]") || html.includes("family");
return {
order_reference: `ORD-${Date.now()}-${Math.floor(Math.random()*1000)}`,
platform,
merchant_name,
order_datetime,
currency,
payment: { credits_amount, card_amount, card_last4 },
totals: {
subtotal,
taxes,
delivery_fee,
service_fee,
tip,
discounts,
total_charged,
},
line_items,
is_family,
};
async function ensureTag(name: string): Promise<number> {
const existing = await queryRow<{ id: number }>(`SELECT id FROM tags WHERE name = $1`, [name]);
if (existing) return existing.id;
const created = await queryRow<{ id: number }>(
`INSERT INTO tags (name, color) VALUES ($1, '#ef4444')
ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id`,
[name]
);
return created!.id;
}
/**
* Validates extraction arithmetic (±$0.02 margin).
*/
export function validateOrderTotals(order: ParsedOrder): boolean {
const { subtotal, taxes = 0, delivery_fee = 0, service_fee = 0, tip = 0, discounts = 0, total_charged } = order.totals;
const calculatedTotal = (subtotal + (taxes || 0) + (delivery_fee || 0) + (service_fee || 0) + (tip || 0)) - (discounts || 0);
if (Math.abs(calculatedTotal - total_charged) > 0.02) {
return false;
}
const { credits_amount = 0, card_amount = 0 } = order.payment;
if (credits_amount !== null || card_amount !== null) {
const paymentSum = (credits_amount || 0) + (card_amount || 0);
if (Math.abs(paymentSum - total_charged) > 0.02) {
return false;
}
}
return true;
}
/**
* Resolves merchant name to category. Never defaults to 'dining' (I4).
*/
export function resolveMerchantCategory(merchantName: string, platform?: string): { category: string; flagReview: boolean } {
if (platform === "uber" || merchantName.toLowerCase().includes("uber trip")) {
return { category: "transport", flagReview: false };
}
const lower = merchantName.toLowerCase();
if (lower.includes("woolworths") || lower.includes("aldi") || lower.includes("coles")) {
return { category: "groceries", flagReview: false };
}
if (lower.includes("mad mex") || lower.includes("guzman") || lower.includes("grill'd")) {
return { category: "dining", flagReview: false };
}
return { category: "other", flagReview: true };
}
/**
* Ingestion runner enforcing DB constraints and invariants (I1-I11).
* Records one parsed order.
*
* Invariants enforced here:
* I1 no credits-funded order before the cutover — before it, splits lived in
* another system and re-importing double-counts.
* I5 a card-settled order creates NO transaction. The statement line is the
* transaction; creating another would double-count.
* I6 a credits-funded order creates a transaction for the credits portion at
* face value.
* I7 idempotent on (source, order_reference).
* I11 [Family] orders are imported and tagged, never silently dropped.
*/
export async function processOrderIngestion(
order: ParsedOrder,
options: { messageId?: string; backfillMode?: boolean } = {}
): Promise<{ transactionId: number | null; metadataId: number; reviewFlag: boolean }> {
// Guard I1: No credits payment before cutover date 2026-01-09
const orderDate = new Date(order.order_datetime);
const cutoverDate = new Date("2026-01-09");
const isCredits = (order.payment.credits_amount || 0) > 0;
): Promise<IngestResult> {
const flags = [...order.flags];
const day = order.order_datetime.slice(0, 10);
if (isCredits && orderDate < cutoverDate) {
return { transactionId: null, metadataId: 0, reviewFlag: false };
}
// Idempotency check I7
const existingMeta = await queryRow<{ id: number; transaction_id: number | null }>(
`SELECT id, transaction_id FROM expense_metadata WHERE source = 'email' AND order_reference = $1`,
// ---- I7: idempotency ----------------------------------------------------
const existing = await queryRow<{ id: number; transaction_id: number | null }>(
`SELECT id, transaction_id FROM expense_metadata
WHERE source = 'email' AND order_reference = $1`,
[order.order_reference]
);
if (existingMeta) {
return { transactionId: existingMeta.transaction_id, metadataId: existingMeta.id, reviewFlag: false };
if (existing) {
return {
transactionId: existing.transaction_id,
metadataId: existing.id,
flags,
skipped: "already_ingested",
};
}
const { category, flagReview } = resolveMerchantCategory(order.merchant_name);
// ---- resolve the credits portion ----------------------------------------
let creditsAmount: number | null = null;
let cardAmount: number | null = order.payment.card_amount;
if (order.payment.ambiguous) {
const { cardAmount: reconciled } = await reconcileCardLeg(order);
if (reconciled === null) {
// No statement line yet. For a live order this is the NORMAL case, not an
// error — card statements arrive monthly, so an order ingested today has
// no card leg in the ledger for weeks (user, 2026-07-26).
//
// Deciding now would mean guessing. Instead the order is recorded as
// provenance with no transaction, and left pending: reconcilePendingOrders()
// resolves it once the statement lands. Backfill hits the same path and
// resolves immediately, because those statements are already imported.
flags.push("awaiting_card_statement");
cardAmount = null;
} else {
cardAmount = reconciled;
const remainder = Number((order.totals.total_charged - reconciled).toFixed(2));
if (remainder > 0.02) {
creditsAmount = remainder;
flags.push(`split_reconciled_card_${reconciled.toFixed(2)}`);
}
}
} else {
creditsAmount = order.payment.credits_amount;
}
// ---- I1: cutover --------------------------------------------------------
if (creditsAmount !== null && day < CUTOVER_DATE) {
return { transactionId: null, metadataId: null, flags, skipped: "pre_cutover" };
}
// ---- I6 / I5 ------------------------------------------------------------
let transactionId: number | null = null;
if (creditsAmount !== null && creditsAmount > 0) {
const isAud = order.currency === "AUD";
if (!isAud) flags.push(`foreign_currency_${order.currency}`);
// I5 & I6: Card-paid orders create NO transaction row. Mixed / credits-only creates transaction for CREDITS portion only.
if (isCredits) {
const creditsPortion = order.payment.credits_amount!;
const txnRow = await queryRow<{ id: number }>(
const txn = await queryRow<{ id: number }>(
`INSERT INTO transactions (
transaction_date, description, amount, category, payment_method, merchant_normalized, owner_id
) VALUES ($1, $2, $3, $4, 'credits', $5, NULL) RETURNING id`,
transaction_date, description, amount, amount_aud, category, payment_method,
merchant_name, merchant_normalized, transaction_type,
foreign_currency_amount, foreign_currency_code, owner_id
) VALUES ($1,$2,$3,$4,$5,'credits',$6,$6,'debit',$7,$8,NULL)
RETURNING id`,
[
order.order_datetime.split("T")[0],
day,
`Order - ${order.merchant_name}`,
creditsPortion,
category,
creditsAmount,
// No FX rate is available at ingest, so amount_aud is left NULL for
// foreign orders rather than asserting a conversion we cannot make.
isAud ? creditsAmount : null,
resolveCategory(order),
order.merchant_name,
isAud ? null : creditsAmount,
isAud ? null : order.currency,
]
);
transactionId = txnRow!.id;
transactionId = txn!.id;
// I11: [Family] orders tagged 'family'
// I11: tag, don't drop. The tag is what removes it from budgets.
if (order.is_family) {
let tagRow = await queryRow<{ id: number }>(`SELECT id FROM tags WHERE name = 'family'`);
if (!tagRow) {
tagRow = (await queryRow<{ id: number }>(`INSERT INTO tags (name, color) VALUES ('family', '#ef4444') RETURNING id`))!;
}
await queryRaw(`INSERT INTO transaction_tags (transaction_id, tag_id) VALUES ($1, $2) ON CONFLICT DO NOTHING`, [
transactionId,
tagRow.id,
]);
const tagId = await ensureTag("family");
await queryRaw(
`INSERT INTO transaction_tags (transaction_id, tag_id) VALUES ($1,$2)
ON CONFLICT DO NOTHING`,
[transactionId, tagId]
);
}
}
// Insert expense_metadata
const metaRow = (await queryRow<{ id: number }>(
// ---- provenance ---------------------------------------------------------
const pending = flags.includes("awaiting_card_statement");
const meta = await queryRow<{ id: number }>(
`INSERT INTO expense_metadata (
transaction_id, source, source_message_id, order_reference, line_items,
subtotal, amount, merchant_normalized, transaction_date
) VALUES ($1, 'email', $2, $3, $4::jsonb, $5, $6, $7, $8) RETURNING id`,
transaction_id, source, source_message_id, order_reference, line_items,
subtotal, amount, merchant_normalized, transaction_date,
card_last4, currency, flags, reconciled_at
) VALUES ($1,'email',$2,$3,$4::jsonb,$5,$6,$7,$8,$9,$10,$11::jsonb,$12)
RETURNING id`,
[
transactionId,
options.messageId || null,
@@ -276,9 +187,122 @@ export async function processOrderIngestion(
order.totals.subtotal,
order.totals.total_charged,
order.merchant_name,
order.order_datetime.split("T")[0],
day,
order.payment.card_last4,
order.currency,
JSON.stringify(flags),
pending ? null : new Date().toISOString(),
]
))!;
);
return { transactionId, metadataId: metaRow.id, reviewFlag: flagReview };
return { transactionId, metadataId: meta!.id, flags };
}
/**
* Second pass over orders parked awaiting a card statement.
*
* Run after each statement import. For every pending order it retries the
* reconciliation; once the card leg appears, any remainder above it was paid in
* credits and becomes a transaction at that point. Orders whose card leg covers
* the whole total resolve to "fully card" and correctly create nothing.
*
* Idempotent: a resolved row gets reconciled_at set and is never revisited.
*/
export async function reconcilePendingOrders(): Promise<{
examined: number;
resolved: number;
created: number;
}> {
const pendingRows = await queryRaw<{
id: number;
order_reference: string;
amount: string;
transaction_date: string;
merchant_normalized: string;
card_last4: string | null;
currency: string | null;
}>(
`SELECT id, order_reference, amount::text, transaction_date::text,
merchant_normalized, card_last4, currency
FROM expense_metadata
WHERE transaction_id IS NULL
AND reconciled_at IS NULL
AND card_last4 IS NOT NULL`
);
let resolved = 0;
let created = 0;
for (const row of pendingRows) {
const total = Number(row.amount);
const probe: ParsedOrder = {
order_reference: row.order_reference,
platform: "doordash",
merchant_name: row.merchant_normalized,
order_datetime: `${row.transaction_date}T00:00:00Z`,
currency: row.currency || "AUD",
payment: { credits_amount: null, card_amount: null, card_last4: row.card_last4, ambiguous: true },
totals: {
subtotal: null, taxes: null, delivery_fee: null,
service_fee: null, tip: null, discounts: null, total_charged: total,
},
line_items: [],
is_family: false,
flags: [],
};
const { cardAmount } = await reconcileCardLeg(probe);
if (cardAmount === null) continue; // statement still hasn't arrived
const remainder = Number((total - cardAmount).toFixed(2));
let txnId: number | null = null;
if (remainder > 0.02 && row.transaction_date >= CUTOVER_DATE) {
const txn = await queryRow<{ id: number }>(
`INSERT INTO transactions (
transaction_date, description, amount, amount_aud, category,
payment_method, merchant_name, merchant_normalized, transaction_type, owner_id
) VALUES ($1,$2,$3,$3,'dining','credits',$4,$4,'debit',NULL)
RETURNING id`,
[row.transaction_date, `Order - ${row.merchant_normalized}`, remainder, row.merchant_normalized]
);
txnId = txn!.id;
created++;
}
await queryRaw(
`UPDATE expense_metadata
SET transaction_id = COALESCE($2, transaction_id),
reconciled_at = NOW(),
flags = flags || $3::jsonb
WHERE id = $1`,
[row.id, txnId, JSON.stringify([`card_leg_${cardAmount.toFixed(2)}`])]
);
resolved++;
}
return { examined: pendingRows.length, resolved, created };
}
/**
* Category from the merchant.
*
* The spec's Correction 1 said never default to `dining`, because ~19% of the
* corpus is groceries and a blanket dining default misfiles a fifth of orders.
* That reasoning is right about groceries and wrong about the remedy: the
* earlier implementation sent everything unrecognised to `other`, and since it
* knew six merchants, that meant Carl's Jr, Taco Bell, Chilli India, Oporto,
* Hungry Jacks, Schnitz, Subway, Souvlaki GR and the rest all landed in
* `other` — worse than the problem it avoided.
*
* Deliberate reversal: grocery merchants are a closed, enumerable set;
* restaurants are an open one. So match groceries explicitly and let the
* residual be `dining`, which is what a delivery order otherwise is. A
* misfiled grocer is one rule away from fixed; a corpus in `other` is not.
*/
export function resolveCategory(order: ParsedOrder): string {
if (order.platform === "uber") return "transport";
const m = order.merchant_name.toLowerCase();
if (/woolworths|aldi|coles|glomark|keells|cargills|iga|costco/.test(m)) return "groceries";
return "dining";
}
+10 -14
View File
@@ -164,26 +164,22 @@ function parsePayment(platform: string, html: string, text: string): PaymentBrea
};
if (platform === "doordash") {
const paid = text.match(/Paid with\s+([^\n]{1,120}?)(?:\s{2,}|$)/i);
const line = paid ? collapse(paid[1]) : "";
// "MasterCard Ending in 8032 and/or credits" — DoorDash names both methods
// and never the split. Resolved as card (user, 2026-07-26: the real charge
// went to 8032), which is also the conservative reading: a card order
// creates no transaction, so it cannot double-count against the statement
// line that will arrive for that card. Flagged either way.
if (/and\/or/i.test(line)) {
// Match the instrument directly. An earlier version captured a trailing
// window delimited by a double space, which does not survive whitespace
// collapsing — so every card/mixed receipt fell through to the credits
// branch and booked the full total as credits.
if (/Paid with[\s\S]{0,60}?and\/or\s*credits/i.test(text)) {
out.ambiguous = true;
const l4 = line.match(/Ending in\s*(\d{3,4})/i);
const l4 = text.match(/Paid with[\s\S]{0,60}?Ending in\s*(\d{3,4})/i);
out.card_last4 = l4 ? l4[1] : null;
return out;
}
if (/credits/i.test(line)) return out; // credits-only; amount filled from total
const l4 = line.match(/Ending in\s*(\d{3,4})/i);
if (l4) {
out.card_last4 = l4[1];
const card = text.match(/Paid with[\s\S]{0,40}?Ending in\s*(\d{3,4})/i);
if (card) {
out.card_last4 = card[1];
return out; // card-only; amount filled from total
}
if (/Paid with\s+credits/i.test(text)) return out; // credits-only
return out;
}