import { queryRaw, queryRow } from "./db"; import type { ParsedOrder } from "./order-parse"; export * from "./order-parse"; export const CUTOVER_DATE = "2026-01-09"; /** * Owner for ingested orders. Analytics scope on COALESCE(t.owner_id, * s.owner_id); an ingested order has no statement, so leaving owner_id NULL * hides it from every view in the app while it sits in the table. */ export const DEFAULT_OWNER_ID = 1; /** * Transaction description. * * The merchant alone ("Order - Burger Corner") does not say where to go and * look for the detail, and there are restaurants on both platforms. The * platform is the one thing the parser always knows and used to discard. */ export function orderDescription(_platform: ParsedOrder["platform"], merchant: string): string { // The platform is deliberately NOT in the headline. What identifies the // transaction is the restaurant; which courier delivered it is provenance, // and it already has a home — the expandable Order details panel renders // `expense_metadata.platform` next to its heading (user, 2026-07-28). // // Keeping it here also fragmented the merchant: the same restaurant reads // "Mad Mex (DoorDash)" and "Mad Mex (Uber Eats)" depending on who carried // the bag, which is not a distinction anyone rating the food cares about. return `Order - ${merchant}`; } export interface IngestResult { transactionId: number | null; metadataId: number | null; flags: string[]; skipped?: string; } /** * Resolves how much of a card-settled order actually hit the card. * * DoorDash writes "MasterCard Ending in 8032 and/or credits" without ever * stating the split. The split is not in the mail — but it IS in the ledger: * the card leg arrives on the statement for that card. Reconciling against it * beats guessing (user, 2026-07-26). * * Subway receipt $29.08, statement 8032 charge $13.06 -> $16.02 was credits * lahori receipt $50.85, statement 8032 charge $50.85 -> fully card * * Returns the card amount if a statement line can be matched, else null. A null * means "unknown", and the caller must not invent a credits figure from it. */ export async function reconcileCardLeg( order: ParsedOrder, windowDays = 4 ): Promise<{ cardAmount: number | null; matchedTransactionId: number | null }> { const last4 = order.payment.card_last4; if (!last4) return { cardAmount: null, matchedTransactionId: null }; const day = order.order_datetime.slice(0, 10); const row = await queryRow<{ id: number; amount: string }>( `SELECT t.id, t.amount::text FROM transactions t JOIN statements s ON s.id = t.statement_id WHERE replace(s.account_number, '-', '') LIKE $1 AND t.transaction_date BETWEEN $2::date - $4::int AND $2::date + $4::int AND (t.description ILIKE '%doordash%' OR t.description ILIKE '%uber%') AND t.amount <= $3::numeric + 0.02 -- A statement line settles exactly one order. Without this, two orders -- on the same card inside the window both match the same charge and -- each books its own credits remainder — double-counting spend. At -- 10-15 orders a month on one card that is not a corner case. AND NOT EXISTS ( SELECT 1 FROM expense_metadata em WHERE em.matched_transaction_id = t.id AND ($5::text IS NULL OR em.order_reference IS DISTINCT FROM $5::text) ) ORDER BY abs(t.amount - $3::numeric), abs(t.transaction_date - $2::date) LIMIT 1`, [`%${last4}`, day, order.totals.total_charged, windowDays, order.order_reference || null] ); if (!row) return { cardAmount: null, matchedTransactionId: null }; return { cardAmount: Number(row.amount), matchedTransactionId: row.id }; } async function ensureTag(name: string): Promise { const existing = await queryRow<{ id: number }>(`SELECT id FROM tags WHERE name = $1`, [name]); if (existing) return existing.id; const created = await queryRow<{ id: number }>( `INSERT INTO tags (name, color) VALUES ($1, '#ef4444') ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id`, [name] ); return created!.id; } /** * Records one parsed order. * * Invariants enforced here: * I1 no credits-funded order before the cutover — before it, splits lived in * another system and re-importing double-counts. * I5 a card-settled order creates NO transaction. The statement line is the * transaction; creating another would double-count. * I6 a credits-funded order creates a transaction for the credits portion at * face value. * I7 idempotent on (source, order_reference). * I11 [Family] orders are imported and tagged, never silently dropped. */ export async function processOrderIngestion( order: ParsedOrder, options: { messageId?: string; backfillMode?: boolean; ownerId?: number; subject?: string; sender?: string; } = {} ): Promise { const flags = [...order.flags]; const day = order.order_datetime.slice(0, 10); // ---- I7: idempotency ---------------------------------------------------- const existing = await queryRow<{ id: number; transaction_id: number | null }>( `SELECT id, transaction_id FROM expense_metadata WHERE source = 'email' AND order_reference = $1`, [order.order_reference] ); if (existing) { return { transactionId: existing.transaction_id, metadataId: existing.id, flags, skipped: "already_ingested", }; } // ---- resolve the credits portion ---------------------------------------- let creditsAmount: number | null = null; if (order.payment.ambiguous) { const { cardAmount: reconciled } = await reconcileCardLeg(order); if (reconciled === null) { // No statement line yet. For a live order this is the NORMAL case, not an // error — card statements arrive monthly, so an order ingested today has // no card leg in the ledger for weeks (user, 2026-07-26). // // Deciding now would mean guessing. Instead the order is recorded as // provenance with no transaction, and left pending: reconcilePendingOrders() // resolves it once the statement lands. Backfill hits the same path and // resolves immediately, because those statements are already imported. flags.push("awaiting_card_statement"); } else { const remainder = Number((order.totals.total_charged - reconciled).toFixed(2)); if (remainder > 0.02) { creditsAmount = remainder; flags.push(`split_reconciled_card_${reconciled.toFixed(2)}`); } } } else { creditsAmount = order.payment.credits_amount; } // ---- I1: cutover (retired 2026-07-28) ----------------------------------- // // This used to return early for any credits order dated before the cutover, // storing NOTHING — no transaction and no metadata, so the receipt was // discarded entirely. // // Its reason was splits, not spend: before 2026-01-09 shared expenses lived // in SplitMyExpenses, and re-importing them would double-charge against // carryover transaction 2348. That reason expired with 788219b, where // ACTIVE_OBLIGATION became `settled = false AND transaction_date >= // '2026-01-09'` — a pre-cutover split can no longer assert a debt, so a // pre-cutover order cannot move a balance however it is recorded. // // What it was left doing was hiding ordinary history: 275 orders, $9,799.96 // of meals and rides across 2020-2025, invisible purely because they were // paid from a gift-card balance instead of a card. Ingestion still writes no // splits at all, so nothing here touches what anyone owes. // // The known imprecision is on the funding side and is accepted rather than // guessed at — see migration 0026 for why the ShopBack loads are left as // they are. if (creditsAmount !== null && day < CUTOVER_DATE) { flags.push("pre_cutover_credits_order"); } // ---- I6 / I5 ------------------------------------------------------------ let transactionId: number | null = null; if (creditsAmount !== null && creditsAmount > 0) { const isAud = order.currency === "AUD"; if (!isAud) flags.push(`foreign_currency_${order.currency}`); const txn = await queryRow<{ id: number }>( `INSERT INTO transactions ( transaction_date, description, amount, amount_aud, category, payment_method, merchant_name, merchant_normalized, transaction_type, foreign_currency_amount, foreign_currency_code, owner_id ) VALUES ($1,$2,$3,$4,$5,'credits',$6,$6,'debit',$7,$8,$9) RETURNING id`, [ day, orderDescription(order.platform, order.merchant_name), creditsAmount, // No FX rate is available at ingest, so amount_aud is left NULL for // foreign orders rather than asserting a conversion we cannot make. isAud ? creditsAmount : null, resolveCategory(order), order.merchant_name, isAud ? null : creditsAmount, isAud ? null : order.currency, // Owner scoping is COALESCE(t.owner_id, s.owner_id). These rows carry // no statement, so a NULL owner_id makes them invisible in every view // in the app — present in the table, absent from the UI. The backfill // inserted 85 rows nobody could see. options.ownerId ?? DEFAULT_OWNER_ID, ] ); transactionId = txn!.id; // I11: tag, don't drop. The tag is what removes it from budgets. if (order.is_family) { const tagId = await ensureTag("family"); await queryRaw( `INSERT INTO transaction_tags (transaction_id, tag_id) VALUES ($1,$2) ON CONFLICT DO NOTHING`, [transactionId, tagId] ); } } // ---- provenance --------------------------------------------------------- const pending = flags.includes("awaiting_card_statement"); const meta = await queryRow<{ id: number }>( `INSERT INTO expense_metadata ( transaction_id, source, source_message_id, order_reference, line_items, subtotal, amount, merchant_normalized, transaction_date, card_last4, currency, flags, reconciled_at, platform, source_email_subject, source_email_from, route ) VALUES ($1,'email',$2,$3,$4::jsonb,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16::jsonb) RETURNING id`, [ transactionId, options.messageId || null, order.order_reference, JSON.stringify(order.line_items), order.totals.subtotal, order.totals.total_charged, order.merchant_name, day, order.payment.card_last4, order.currency, JSON.stringify(flags), pending ? null : new Date().toISOString(), order.platform, options.subject ?? null, options.sender ?? null, JSON.stringify(order.route ?? []), ] ); return { transactionId, metadataId: meta!.id, flags }; } /** * Second pass over orders parked awaiting a card statement. * * Run after each statement import. For every pending order it retries the * reconciliation; once the card leg appears, any remainder above it was paid in * credits and becomes a transaction at that point. Orders whose card leg covers * the whole total resolve to "fully card" and correctly create nothing. * * Idempotent: a resolved row gets reconciled_at set and is never revisited. */ export async function reconcilePendingOrders(): Promise<{ examined: number; resolved: number; created: number; }> { const pendingRows = await queryRaw<{ id: number; order_reference: string; amount: string; transaction_date: string; merchant_normalized: string; card_last4: string | null; currency: string | null; platform: ParsedOrder["platform"] | null; }>( `SELECT id, order_reference, amount::text, transaction_date::text, merchant_normalized, card_last4, currency, platform FROM expense_metadata WHERE transaction_id IS NULL AND reconciled_at IS NULL AND card_last4 IS NOT NULL` ); let resolved = 0; let created = 0; for (const row of pendingRows) { const total = Number(row.amount); const probe: ParsedOrder = { order_reference: row.order_reference, platform: row.platform ?? "doordash", merchant_name: row.merchant_normalized, order_datetime: `${row.transaction_date}T00:00:00Z`, currency: row.currency || "AUD", payment: { credits_amount: null, card_amount: null, card_last4: row.card_last4, ambiguous: true }, totals: { subtotal: null, taxes: null, delivery_fee: null, service_fee: null, tip: null, discounts: null, total_charged: total, }, line_items: [], route: [], is_family: false, flags: [], }; const { cardAmount, matchedTransactionId } = await reconcileCardLeg(probe); if (cardAmount === null) continue; // statement still hasn't arrived const remainder = Number((total - cardAmount).toFixed(2)); let txnId: number | null = null; if (remainder > 0.02 && row.transaction_date >= CUTOVER_DATE) { // Category from the merchant, never hardcoded. Hardcoding 'dining' here // silently misfiled every grocery order that arrived with an unstated // split — reintroducing, through the deferred path, exactly the // misfiling resolveCategory() exists to prevent. const category = resolveCategory({ merchant_name: row.merchant_normalized, platform: "doordash", } as ParsedOrder); const txn = await queryRow<{ id: number }>( `INSERT INTO transactions ( transaction_date, description, amount, amount_aud, category, payment_method, merchant_name, merchant_normalized, transaction_type, owner_id ) VALUES ($1,$2,$3,$3,$5,'credits',$4,$4,'debit',$6) RETURNING id`, [row.transaction_date, orderDescription(row.platform ?? "doordash", row.merchant_normalized), remainder, row.merchant_normalized, category, DEFAULT_OWNER_ID] ); txnId = txn!.id; created++; } await queryRaw( `UPDATE expense_metadata SET transaction_id = COALESCE($2, transaction_id), matched_transaction_id = $4, reconciled_at = NOW(), flags = flags || $3::jsonb WHERE id = $1`, [row.id, txnId, JSON.stringify([`card_leg_${cardAmount.toFixed(2)}`]), matchedTransactionId] ); resolved++; } return { examined: pendingRows.length, resolved, created }; } /** * Category from the merchant. * * The spec's Correction 1 said never default to `dining`, because ~19% of the * corpus is groceries and a blanket dining default misfiles a fifth of orders. * That reasoning is right about groceries and wrong about the remedy: the * earlier implementation sent everything unrecognised to `other`, and since it * knew six merchants, that meant Carl's Jr, Taco Bell, Chilli India, Oporto, * Hungry Jacks, Schnitz, Subway, Souvlaki GR and the rest all landed in * `other` — worse than the problem it avoided. * * Deliberate reversal: grocery merchants are a closed, enumerable set; * restaurants are an open one. So match groceries explicitly and let the * residual be `dining`, which is what a delivery order otherwise is. A * misfiled grocer is one rule away from fixed; a corpus in `other` is not. */ export function resolveCategory(order: ParsedOrder): string { if (order.platform === "uber") return "transport"; const m = order.merchant_name.toLowerCase(); if (/woolworths|aldi|coles|glomark|keells|cargills|iga|costco/.test(m)) return "groceries"; return "dining"; } /** * Applies a refund / total-adjustment notice to an already-ingested order. * * The order it amends is matched by the UUID Uber reuses across the original * receipt and the amendment. The recorded transaction is reduced to the new * total rather than a compensating negative row being added: the order is one * event and its cost changed, and a second row would misreport both the meal * count and the merchant's spend. * * If the original has not been ingested (amendment arrived first, or the * receipt predates the cutover), nothing is invented — the amendment is * recorded as unmatched for a later pass. */ export async function applyOrderAmendment(a: { order_reference: string | null; new_total: number; refund_amount: number | null; previous_total: number | null; order_datetime: string; messageId: string; }): Promise<{ matched: boolean; transactionId: number | null; adjusted: number | null }> { if (!a.order_reference) return { matched: false, transactionId: null, adjusted: null }; const meta = await queryRow<{ id: number; transaction_id: number | null; amount: string }>( `SELECT id, transaction_id, amount::text FROM expense_metadata WHERE source = 'email' AND order_reference = $1`, [a.order_reference] ); if (!meta) return { matched: false, transactionId: null, adjusted: null }; await queryRaw( `UPDATE expense_metadata SET amount = $2, flags = flags || $3::jsonb WHERE id = $1`, [ meta.id, a.new_total, JSON.stringify([ `amended_from_${Number(meta.amount).toFixed(2)}`, a.refund_amount !== null ? `refund_${a.refund_amount.toFixed(2)}` : "amended", ]), ] ); if (meta.transaction_id === null) { // Card-settled or still pending: no transaction of ours to reduce. The // refund will show on the statement in its own right. return { matched: true, transactionId: null, adjusted: null }; } await queryRaw( `UPDATE transactions SET amount = $2, amount_aud = CASE WHEN foreign_currency_code IS NULL THEN $2 ELSE amount_aud END WHERE id = $1`, [meta.transaction_id, a.new_total] ); return { matched: true, transactionId: meta.transaction_id, adjusted: a.new_total }; }