\s*Total:/i) ||
- html.match(/order with\s+([A-Za-z0-9\s'&-]+)/i) ||
- html.match(/
\s*([A-Za-z0-9\s'&-]+)\s*<\/div>/i);
- merchant_name = merchantMatch ? merchantMatch[1].trim() : "Uber Eats Merchant";
- }
-
- // Parse line items (Uber Eats has line_items: [] per Difference 1; Uber Rides parse fare components per Difference 2)
- const line_items: LineItem[] = [];
- if (platform === "uber") {
- // Uber Ride fare components
- const tripFare = extractTableValue(/Trip fare/i);
- if (tripFare) line_items.push({ qty: 1, description: "Trip fare", amount: tripFare });
- const bookingFee = extractTableValue(/Booking Fee/i);
- if (bookingFee) line_items.push({ qty: 1, description: "Booking Fee", amount: bookingFee });
- const airportFee = extractTableValue(/Airport fee/i);
- if (airportFee) line_items.push({ qty: 1, description: "Airport fee", amount: airportFee });
- } else if (platform === "doordash") {
- const lineItemMatch = html.match(/(\d+)x\s+([^$]+)\s+\$(\d+\.\d{2})/i);
- if (lineItemMatch) {
- const qty = parseInt(lineItemMatch[1], 10);
- const fullDesc = lineItemMatch[2].trim();
- const parts = fullDesc.split('•').map(p => p.trim());
- const description = parts[0];
- const options = parts.slice(1);
- const amount = parseFloat(lineItemMatch[3]);
- line_items.push({ qty, description, amount, options });
- }
- }
-
- // Currency extraction
- const currencyMatch = html.match(/\b(AUD|NZD|LKR|USD)\b/i);
- const currency = currencyMatch ? currencyMatch[1].toUpperCase() : "AUD";
-
- // Date parsing
- const dateMatch = html.match(/Date:\s*(\d{4}-\d{2}-\d{2})/i);
- const order_datetime = dateMatch ? `${dateMatch[1]}T12:00:00Z` : "2026-01-15T12:00:00Z";
-
- // Check if family order (Difference 4)
- const is_family = html.includes("[Family]") || html.includes("family");
-
- return {
- order_reference: `ORD-${Date.now()}-${Math.floor(Math.random()*1000)}`,
- platform,
- merchant_name,
- order_datetime,
- currency,
- payment: { credits_amount, card_amount, card_last4 },
- totals: {
- subtotal,
- taxes,
- delivery_fee,
- service_fee,
- tip,
- discounts,
- total_charged,
- },
- line_items,
- is_family,
- };
+async function ensureTag(name: string): Promise {
+ const existing = await queryRow<{ id: number }>(`SELECT id FROM tags WHERE name = $1`, [name]);
+ if (existing) return existing.id;
+ const created = await queryRow<{ id: number }>(
+ `INSERT INTO tags (name, color) VALUES ($1, '#ef4444')
+ ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id`,
+ [name]
+ );
+ return created!.id;
}
/**
- * Validates extraction arithmetic (±$0.02 margin).
- */
-export function validateOrderTotals(order: ParsedOrder): boolean {
- const { subtotal, taxes = 0, delivery_fee = 0, service_fee = 0, tip = 0, discounts = 0, total_charged } = order.totals;
- const calculatedTotal = (subtotal + (taxes || 0) + (delivery_fee || 0) + (service_fee || 0) + (tip || 0)) - (discounts || 0);
-
- if (Math.abs(calculatedTotal - total_charged) > 0.02) {
- return false;
- }
-
- const { credits_amount = 0, card_amount = 0 } = order.payment;
- if (credits_amount !== null || card_amount !== null) {
- const paymentSum = (credits_amount || 0) + (card_amount || 0);
- if (Math.abs(paymentSum - total_charged) > 0.02) {
- return false;
- }
- }
-
- return true;
-}
-
-/**
- * Resolves merchant name to category. Never defaults to 'dining' (I4).
- */
-export function resolveMerchantCategory(merchantName: string, platform?: string): { category: string; flagReview: boolean } {
- if (platform === "uber" || merchantName.toLowerCase().includes("uber trip")) {
- return { category: "transport", flagReview: false };
- }
- const lower = merchantName.toLowerCase();
- if (lower.includes("woolworths") || lower.includes("aldi") || lower.includes("coles")) {
- return { category: "groceries", flagReview: false };
- }
- if (lower.includes("mad mex") || lower.includes("guzman") || lower.includes("grill'd")) {
- return { category: "dining", flagReview: false };
- }
- return { category: "other", flagReview: true };
-}
-
-/**
- * Ingestion runner enforcing DB constraints and invariants (I1-I11).
+ * Records one parsed order.
+ *
+ * Invariants enforced here:
+ * I1 no credits-funded order before the cutover — before it, splits lived in
+ * another system and re-importing double-counts.
+ * I5 a card-settled order creates NO transaction. The statement line is the
+ * transaction; creating another would double-count.
+ * I6 a credits-funded order creates a transaction for the credits portion at
+ * face value.
+ * I7 idempotent on (source, order_reference).
+ * I11 [Family] orders are imported and tagged, never silently dropped.
*/
export async function processOrderIngestion(
order: ParsedOrder,
options: { messageId?: string; backfillMode?: boolean } = {}
-): Promise<{ transactionId: number | null; metadataId: number; reviewFlag: boolean }> {
- // Guard I1: No credits payment before cutover date 2026-01-09
- const orderDate = new Date(order.order_datetime);
- const cutoverDate = new Date("2026-01-09");
- const isCredits = (order.payment.credits_amount || 0) > 0;
+): Promise {
+ const flags = [...order.flags];
+ const day = order.order_datetime.slice(0, 10);
- if (isCredits && orderDate < cutoverDate) {
- return { transactionId: null, metadataId: 0, reviewFlag: false };
- }
-
- // Idempotency check I7
- const existingMeta = await queryRow<{ id: number; transaction_id: number | null }>(
- `SELECT id, transaction_id FROM expense_metadata WHERE source = 'email' AND order_reference = $1`,
+ // ---- I7: idempotency ----------------------------------------------------
+ const existing = await queryRow<{ id: number; transaction_id: number | null }>(
+ `SELECT id, transaction_id FROM expense_metadata
+ WHERE source = 'email' AND order_reference = $1`,
[order.order_reference]
);
- if (existingMeta) {
- return { transactionId: existingMeta.transaction_id, metadataId: existingMeta.id, reviewFlag: false };
+ if (existing) {
+ return {
+ transactionId: existing.transaction_id,
+ metadataId: existing.id,
+ flags,
+ skipped: "already_ingested",
+ };
}
- const { category, flagReview } = resolveMerchantCategory(order.merchant_name);
+ // ---- resolve the credits portion ----------------------------------------
+ let creditsAmount: number | null = null;
+ let cardAmount: number | null = order.payment.card_amount;
+ if (order.payment.ambiguous) {
+ const { cardAmount: reconciled } = await reconcileCardLeg(order);
+ if (reconciled === null) {
+ // No statement line yet. For a live order this is the NORMAL case, not an
+ // error — card statements arrive monthly, so an order ingested today has
+ // no card leg in the ledger for weeks (user, 2026-07-26).
+ //
+ // Deciding now would mean guessing. Instead the order is recorded as
+ // provenance with no transaction, and left pending: reconcilePendingOrders()
+ // resolves it once the statement lands. Backfill hits the same path and
+ // resolves immediately, because those statements are already imported.
+ flags.push("awaiting_card_statement");
+ cardAmount = null;
+ } else {
+ cardAmount = reconciled;
+ const remainder = Number((order.totals.total_charged - reconciled).toFixed(2));
+ if (remainder > 0.02) {
+ creditsAmount = remainder;
+ flags.push(`split_reconciled_card_${reconciled.toFixed(2)}`);
+ }
+ }
+ } else {
+ creditsAmount = order.payment.credits_amount;
+ }
+
+ // ---- I1: cutover --------------------------------------------------------
+ if (creditsAmount !== null && day < CUTOVER_DATE) {
+ return { transactionId: null, metadataId: null, flags, skipped: "pre_cutover" };
+ }
+
+ // ---- I6 / I5 ------------------------------------------------------------
let transactionId: number | null = null;
+ if (creditsAmount !== null && creditsAmount > 0) {
+ const isAud = order.currency === "AUD";
+ if (!isAud) flags.push(`foreign_currency_${order.currency}`);
- // I5 & I6: Card-paid orders create NO transaction row. Mixed / credits-only creates transaction for CREDITS portion only.
- if (isCredits) {
- const creditsPortion = order.payment.credits_amount!;
- const txnRow = await queryRow<{ id: number }>(
+ const txn = await queryRow<{ id: number }>(
`INSERT INTO transactions (
- transaction_date, description, amount, category, payment_method, merchant_normalized, owner_id
- ) VALUES ($1, $2, $3, $4, 'credits', $5, NULL) RETURNING id`,
+ transaction_date, description, amount, amount_aud, category, payment_method,
+ merchant_name, merchant_normalized, transaction_type,
+ foreign_currency_amount, foreign_currency_code, owner_id
+ ) VALUES ($1,$2,$3,$4,$5,'credits',$6,$6,'debit',$7,$8,NULL)
+ RETURNING id`,
[
- order.order_datetime.split("T")[0],
+ day,
`Order - ${order.merchant_name}`,
- creditsPortion,
- category,
+ creditsAmount,
+ // No FX rate is available at ingest, so amount_aud is left NULL for
+ // foreign orders rather than asserting a conversion we cannot make.
+ isAud ? creditsAmount : null,
+ resolveCategory(order),
order.merchant_name,
+ isAud ? null : creditsAmount,
+ isAud ? null : order.currency,
]
);
- transactionId = txnRow!.id;
+ transactionId = txn!.id;
- // I11: [Family] orders tagged 'family'
+ // I11: tag, don't drop. The tag is what removes it from budgets.
if (order.is_family) {
- let tagRow = await queryRow<{ id: number }>(`SELECT id FROM tags WHERE name = 'family'`);
- if (!tagRow) {
- tagRow = (await queryRow<{ id: number }>(`INSERT INTO tags (name, color) VALUES ('family', '#ef4444') RETURNING id`))!;
- }
- await queryRaw(`INSERT INTO transaction_tags (transaction_id, tag_id) VALUES ($1, $2) ON CONFLICT DO NOTHING`, [
- transactionId,
- tagRow.id,
- ]);
+ const tagId = await ensureTag("family");
+ await queryRaw(
+ `INSERT INTO transaction_tags (transaction_id, tag_id) VALUES ($1,$2)
+ ON CONFLICT DO NOTHING`,
+ [transactionId, tagId]
+ );
}
}
- // Insert expense_metadata
- const metaRow = (await queryRow<{ id: number }>(
+ // ---- provenance ---------------------------------------------------------
+ const pending = flags.includes("awaiting_card_statement");
+ const meta = await queryRow<{ id: number }>(
`INSERT INTO expense_metadata (
- transaction_id, source, source_message_id, order_reference, line_items,
- subtotal, amount, merchant_normalized, transaction_date
- ) VALUES ($1, 'email', $2, $3, $4::jsonb, $5, $6, $7, $8) RETURNING id`,
+ transaction_id, source, source_message_id, order_reference, line_items,
+ subtotal, amount, merchant_normalized, transaction_date,
+ card_last4, currency, flags, reconciled_at
+ ) VALUES ($1,'email',$2,$3,$4::jsonb,$5,$6,$7,$8,$9,$10,$11::jsonb,$12)
+ RETURNING id`,
[
transactionId,
options.messageId || null,
@@ -276,9 +187,122 @@ export async function processOrderIngestion(
order.totals.subtotal,
order.totals.total_charged,
order.merchant_name,
- order.order_datetime.split("T")[0],
+ day,
+ order.payment.card_last4,
+ order.currency,
+ JSON.stringify(flags),
+ pending ? null : new Date().toISOString(),
]
- ))!;
+ );
- return { transactionId, metadataId: metaRow.id, reviewFlag: flagReview };
+ return { transactionId, metadataId: meta!.id, flags };
+}
+
+/**
+ * Second pass over orders parked awaiting a card statement.
+ *
+ * Run after each statement import. For every pending order it retries the
+ * reconciliation; once the card leg appears, any remainder above it was paid in
+ * credits and becomes a transaction at that point. Orders whose card leg covers
+ * the whole total resolve to "fully card" and correctly create nothing.
+ *
+ * Idempotent: a resolved row gets reconciled_at set and is never revisited.
+ */
+export async function reconcilePendingOrders(): Promise<{
+ examined: number;
+ resolved: number;
+ created: number;
+}> {
+ const pendingRows = await queryRaw<{
+ id: number;
+ order_reference: string;
+ amount: string;
+ transaction_date: string;
+ merchant_normalized: string;
+ card_last4: string | null;
+ currency: string | null;
+ }>(
+ `SELECT id, order_reference, amount::text, transaction_date::text,
+ merchant_normalized, card_last4, currency
+ FROM expense_metadata
+ WHERE transaction_id IS NULL
+ AND reconciled_at IS NULL
+ AND card_last4 IS NOT NULL`
+ );
+
+ let resolved = 0;
+ let created = 0;
+
+ for (const row of pendingRows) {
+ const total = Number(row.amount);
+ const probe: ParsedOrder = {
+ order_reference: row.order_reference,
+ platform: "doordash",
+ merchant_name: row.merchant_normalized,
+ order_datetime: `${row.transaction_date}T00:00:00Z`,
+ currency: row.currency || "AUD",
+ payment: { credits_amount: null, card_amount: null, card_last4: row.card_last4, ambiguous: true },
+ totals: {
+ subtotal: null, taxes: null, delivery_fee: null,
+ service_fee: null, tip: null, discounts: null, total_charged: total,
+ },
+ line_items: [],
+ is_family: false,
+ flags: [],
+ };
+
+ const { cardAmount } = await reconcileCardLeg(probe);
+ if (cardAmount === null) continue; // statement still hasn't arrived
+
+ const remainder = Number((total - cardAmount).toFixed(2));
+ let txnId: number | null = null;
+
+ if (remainder > 0.02 && row.transaction_date >= CUTOVER_DATE) {
+ const txn = await queryRow<{ id: number }>(
+ `INSERT INTO transactions (
+ transaction_date, description, amount, amount_aud, category,
+ payment_method, merchant_name, merchant_normalized, transaction_type, owner_id
+ ) VALUES ($1,$2,$3,$3,'dining','credits',$4,$4,'debit',NULL)
+ RETURNING id`,
+ [row.transaction_date, `Order - ${row.merchant_normalized}`, remainder, row.merchant_normalized]
+ );
+ txnId = txn!.id;
+ created++;
+ }
+
+ await queryRaw(
+ `UPDATE expense_metadata
+ SET transaction_id = COALESCE($2, transaction_id),
+ reconciled_at = NOW(),
+ flags = flags || $3::jsonb
+ WHERE id = $1`,
+ [row.id, txnId, JSON.stringify([`card_leg_${cardAmount.toFixed(2)}`])]
+ );
+ resolved++;
+ }
+
+ return { examined: pendingRows.length, resolved, created };
+}
+
+/**
+ * Category from the merchant.
+ *
+ * The spec's Correction 1 said never default to `dining`, because ~19% of the
+ * corpus is groceries and a blanket dining default misfiles a fifth of orders.
+ * That reasoning is right about groceries and wrong about the remedy: the
+ * earlier implementation sent everything unrecognised to `other`, and since it
+ * knew six merchants, that meant Carl's Jr, Taco Bell, Chilli India, Oporto,
+ * Hungry Jacks, Schnitz, Subway, Souvlaki GR and the rest all landed in
+ * `other` — worse than the problem it avoided.
+ *
+ * Deliberate reversal: grocery merchants are a closed, enumerable set;
+ * restaurants are an open one. So match groceries explicitly and let the
+ * residual be `dining`, which is what a delivery order otherwise is. A
+ * misfiled grocer is one rule away from fixed; a corpus in `other` is not.
+ */
+export function resolveCategory(order: ParsedOrder): string {
+ if (order.platform === "uber") return "transport";
+ const m = order.merchant_name.toLowerCase();
+ if (/woolworths|aldi|coles|glomark|keells|cargills|iga|costco/.test(m)) return "groceries";
+ return "dining";
}
diff --git a/src/lib/order-parse.ts b/src/lib/order-parse.ts
index 775c983..cd50817 100644
--- a/src/lib/order-parse.ts
+++ b/src/lib/order-parse.ts
@@ -164,26 +164,22 @@ function parsePayment(platform: string, html: string, text: string): PaymentBrea
};
if (platform === "doordash") {
- const paid = text.match(/Paid with\s+([^\n]{1,120}?)(?:\s{2,}|$)/i);
- const line = paid ? collapse(paid[1]) : "";
-
- // "MasterCard Ending in 8032 and/or credits" — DoorDash names both methods
- // and never the split. Resolved as card (user, 2026-07-26: the real charge
- // went to 8032), which is also the conservative reading: a card order
- // creates no transaction, so it cannot double-count against the statement
- // line that will arrive for that card. Flagged either way.
- if (/and\/or/i.test(line)) {
+ // Match the instrument directly. An earlier version captured a trailing
+ // window delimited by a double space, which does not survive whitespace
+ // collapsing — so every card/mixed receipt fell through to the credits
+ // branch and booked the full total as credits.
+ if (/Paid with[\s\S]{0,60}?and\/or\s*credits/i.test(text)) {
out.ambiguous = true;
- const l4 = line.match(/Ending in\s*(\d{3,4})/i);
+ const l4 = text.match(/Paid with[\s\S]{0,60}?Ending in\s*(\d{3,4})/i);
out.card_last4 = l4 ? l4[1] : null;
return out;
}
- if (/credits/i.test(line)) return out; // credits-only; amount filled from total
- const l4 = line.match(/Ending in\s*(\d{3,4})/i);
- if (l4) {
- out.card_last4 = l4[1];
+ const card = text.match(/Paid with[\s\S]{0,40}?Ending in\s*(\d{3,4})/i);
+ if (card) {
+ out.card_last4 = card[1];
return out; // card-only; amount filled from total
}
+ if (/Paid with\s+credits/i.test(text)) return out; // credits-only
return out;
}