From c25db5211bac6fbac1ac1f0dad788cc5d5ba2a33 Mon Sep 17 00:00:00 2001 From: siddharthd Date: Fri, 21 Aug 2026 17:53:06 +1000 Subject: [PATCH] intake: a NULL platform is "no platform", and Create re-checks before it inserts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six fixes in the router, all of them cases where a stale or over-wide answer was written rather than re-asked. - ($3 IS NULL OR em.platform = $3) made an in-store receipt a wildcard: a till receipt from the Coles tap matched the Uber Eats Coles order for the same day and money, and overwrote its items. In-store now matches only rows with no platform of their own. - findCandidates loses its platform filter entirely. That card is a human choice, shown precisely because the exact key missed — the platform is one of the things that may be wrong, so the list must widen there, not narrow. - createFromBatch re-runs the match. The card is minutes old by the time it is tapped and the email/order bridge books orders on its own schedule; the match it was drawn from is not current. - ...and checks for a resolved sibling batch. findDuplicateBatch ignores unresolved batches by design, so two pending cards for one order each booked it. - INSERT ... ON CONFLICT (source, source_ref) DO NOTHING with a re-select, rather than SELECT-then-INSERT, which loses a double tap. - A non-AUD receipt is booked as order-ingestion.ts books one: amount native, amount_aud NULL, foreign_currency_* set. No FX rate exists at intake. Also: EXCLUDE_RECONCILED_SOURCE in both queries instead of a bare superseded_by_id check — a reconciled manual twin at the exact key read as an ambiguity, which is the same spend counted twice. And routeFood refuses to replace an itemised receipt with an extraction carrying no prices at all: the reconciliation check passes those (there is nothing to check), so without this a priceless upload silently destroys a good record. It holds instead. RouteContext.subtotal was written and never read; it is replaced by currency, which the cards need. --- src/lib/intake-actions.ts | 3 +- src/lib/intake-router.ts | 180 ++++++++++++++++++++++++++++++-------- 2 files changed, 144 insertions(+), 39 deletions(-) diff --git a/src/lib/intake-actions.ts b/src/lib/intake-actions.ts index b7d8970..e30f9a6 100644 --- a/src/lib/intake-actions.ts +++ b/src/lib/intake-actions.ts @@ -58,6 +58,7 @@ const ctxFor = (batch: BatchRow): RouteContext | null => platform: batch.header.platform, header: batch.header, items: batch.items ?? [], + currency: batch.header.currency, } : null; @@ -85,7 +86,7 @@ export async function handleIntakeAction( [batch.id, txnId] ); return { - ...enrichedCard({ batchId: batch.id, merchant: ctx.header.merchant, total: ctx.header.total, orderDate: ctx.header.order_date, platform: ctx.platform, itemCount: ctx.items.length, transactionId: txnId }), + ...enrichedCard({ batchId: batch.id, merchant: ctx.header.merchant, total: ctx.header.total, orderDate: ctx.header.order_date, platform: ctx.platform, itemCount: ctx.items.length, currency: ctx.currency, transactionId: txnId }), replace: true, }; } diff --git a/src/lib/intake-router.ts b/src/lib/intake-router.ts index 90a3164..5f33c0b 100644 --- a/src/lib/intake-router.ts +++ b/src/lib/intake-router.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { queryRaw, queryRow } from "@/lib/db"; +import { EXCLUDE_RECONCILED_SOURCE } from "@/lib/analytics-sql"; import { DEFAULT_OWNER_ID } from "@/lib/order-ingestion"; import { classifyInput, @@ -207,15 +208,31 @@ export async function findDuplicateBatch( return rows.find((r) => merchantMatches(r.header?.merchant, header.merchant)) ?? null; } -type MatchRow = { transaction_id: number; description: string | null; transaction_date: string; amount: string; merchant: string | null; platform: string | null }; +type MatchRow = { + transaction_id: number; + description: string | null; + transaction_date: string; + amount: string; + merchant: string | null; + platform: string | null; + /** The `expense_metadata` row the match resolved through — null when there is none. */ + metadata_id: number | null; + /** Does that row already carry a receipt? */ + has_line_items: boolean; +}; /** * The match rule (plan §5, D2): `(platform, merchant, date, total)`, amount and - * date EXACT, never widened silently. Platform is required when the source - * implies one; an in-store receipt has none and matches on the other three. - * The merchant compares normalised, with containment ("Coles" vs "Coles - * (Wyndham Vale)") — that is spelling, not widening. A unique hit is a match; - * two hits are an ambiguity and go to the candidates card like a miss. + * date EXACT, never widened silently. The merchant compares normalised, with + * whole-word containment ("Coles" vs "Coles (Wyndham Vale)") — that is + * spelling, not widening. A unique hit is a match; two hits are an ambiguity + * and go to the candidates card like a miss. + * + * A NULL platform means "no platform", not "any platform". An in-store Coles + * receipt carries none, and `$3 IS NULL OR em.platform = $3` let it match an + * Uber Eats Coles order on the same day for the same money — and then overwrite + * that order's items with the till receipt's. The in-store side matches rows + * with no platform of their own; the delivery side still requires its own. */ export async function matchTransaction(header: OrderHeader, platform: string | null): Promise<{ match: MatchRow | null; ambiguous: boolean }> { if (!header.merchant) return { match: null, ambiguous: false }; @@ -223,13 +240,14 @@ export async function matchTransaction(header: OrderHeader, platform: string | n `SELECT DISTINCT ON (t.id) t.id AS transaction_id, t.description, t.transaction_date::text, t.amount::text, COALESCE(em.merchant_normalized, t.merchant_normalized, t.merchant_name, t.description) AS merchant, - em.platform + em.platform, em.id AS metadata_id, + COALESCE(jsonb_typeof(em.line_items) = 'array' AND jsonb_array_length(em.line_items) > 0, false) AS has_line_items FROM transactions t LEFT JOIN expense_metadata em ON em.transaction_id = t.id OR em.matched_transaction_id = t.id - WHERE t.superseded_by_id IS NULL + WHERE ${EXCLUDE_RECONCILED_SOURCE} AND t.transaction_date = $1::date AND t.amount = $2::numeric - AND ($3::text IS NULL OR em.platform = $3) + AND (($3::text IS NULL AND em.platform IS NULL) OR em.platform = $3) ORDER BY t.id, (em.transaction_id = t.id) DESC NULLS LAST`, [header.order_date, header.total, platform] ); @@ -238,21 +256,28 @@ export async function matchTransaction(header: OrderHeader, platform: string | n return { match: null, ambiguous: hits.length > 1 }; } -/** Nearest candidates for the card: same platform and merchant, ±3 days, any amount. */ -export async function findCandidates(header: OrderHeader, platform: string | null): Promise { +/** + * Nearest candidates for the card: same merchant, ±3 days, any amount. + * + * Deliberately NOT filtered by platform. The card is a human choice, not a + * decision — the whole reason it is shown is that the exact key missed, so the + * platform is one of the things that may be wrong. A filter here narrows the + * list at exactly the moment it needs to widen, and the row the user was + * looking for is simply absent with nothing saying why. + */ +export async function findCandidates(header: OrderHeader): Promise { if (!header.merchant) return []; const rows = await queryRaw( `SELECT DISTINCT ON (t.id) t.id AS transaction_id, t.description, t.transaction_date::text, t.amount::text, COALESCE(em.merchant_normalized, t.merchant_normalized, t.merchant_name, t.description) AS merchant, - em.platform + em.platform, em.id AS metadata_id, false AS has_line_items FROM transactions t LEFT JOIN expense_metadata em ON em.transaction_id = t.id OR em.matched_transaction_id = t.id - WHERE t.superseded_by_id IS NULL + WHERE ${EXCLUDE_RECONCILED_SOURCE} AND t.transaction_date BETWEEN $1::date - 3 AND $1::date + 3 - AND ($2::text IS NULL OR em.platform = $2) ORDER BY t.id, (em.transaction_id = t.id) DESC NULLS LAST`, - [header.order_date, platform] + [header.order_date] ); return rows .filter((r) => merchantMatches(r.merchant, header.merchant) || merchantMatches(r.description, header.merchant)) @@ -275,7 +300,7 @@ export type RouteContext = { platform: string | null; header: OrderHeader; items: StoredLineItem[]; - subtotal?: number | null; + currency?: string | null; }; const cardInput = (ctx: RouteContext): CardInput => ({ @@ -285,6 +310,7 @@ const cardInput = (ctx: RouteContext): CardInput => ({ orderDate: ctx.header.order_date, platform: ctx.platform, itemCount: ctx.items.length, + currency: ctx.currency, }); const finish = (ctx: RouteContext, kind: IntakeKind, status: IntakeStatus, card: { blocks: Block[]; text: string }, extra: Partial = {}): IntakeResult => ({ @@ -299,15 +325,33 @@ const finish = (ctx: RouteContext, kind: IntakeKind, status: IntakeStatus, card: ...extra, }); +/** + * Are these items unverifiable — no line carrying any money at all? + * + * That is `reconcileItems`' `basis: "none"`: pasted text with no prices, or an + * extraction that read the names and none of the figures. The reconciliation + * check passes it, correctly, because there is nothing to check. What must not + * follow is replacing a receipt that IS itemised with one nothing verified. + */ +const unverifiable = (items: StoredLineItem[]) => items.every((i) => i.amount === null); + /** Food: enrich the matched order's line_items, or show candidates. NEVER auto-creates. */ export async function routeFood(ctx: RouteContext): Promise { const { match } = await matchTransaction(ctx.header, ctx.platform); if (match) { - await replaceLineItems(match.transaction_id, ctx.items, { source: "intake", platform: ctx.platform, batchId: ctx.batchId }); + if (unverifiable(ctx.items) && match.has_line_items) { + await updateBatch(ctx.batchId, { status: "held", items: ctx.items }); + const card = heldCard( + "this upload carries no prices, so nothing checked it against the total — and that order already has its items", + "I left the existing items alone" + ); + return finish(ctx, "held", "held", card, { transaction_id: match.transaction_id }); + } + await replaceLineItems(match.transaction_id, ctx.items, { source: "intake", platform: ctx.platform, batchId: ctx.batchId, metadataId: match.metadata_id }); await updateBatch(ctx.batchId, { kind: "food", status: "enriched", transaction_id: match.transaction_id, items: ctx.items }); return finish(ctx, "food", "enriched", enrichedCard({ ...cardInput(ctx), transactionId: match.transaction_id }), { transaction_id: match.transaction_id }); } - const candidates = await findCandidates(ctx.header, ctx.platform); + const candidates = await findCandidates(ctx.header); await updateBatch(ctx.batchId, { kind: "food", status: "candidates", items: ctx.items }); return finish(ctx, "food", "candidates", candidatesCard(cardInput(ctx), candidates), { candidates }); } @@ -330,7 +374,7 @@ export async function routeGrocery(ctx: RouteContext, files: StoredFile[], pantr } catch (e) { if (e instanceof PantryDuplicateError) { await updateBatch(ctx.batchId, { kind: "grocery", status: "duplicate", capture_id: e.captureId, items: ctx.items }); - return finish(ctx, "grocery", "duplicate", duplicateCard({ merchant: ctx.header.merchant, total: ctx.header.total, transactionId: match?.transaction_id ?? null, captureId: e.captureId, batchId: null }), { + return finish(ctx, "grocery", "duplicate", duplicateCard({ merchant: ctx.header.merchant, total: ctx.header.total, currency: ctx.currency, transactionId: match?.transaction_id ?? null, captureId: e.captureId, batchId: null }), { capture_id: e.captureId ?? undefined, }); } @@ -345,38 +389,98 @@ export async function createFromBatch(batch: BatchRow, kind: "grocery" | "food") if (!batch.header) throw new Error("batch has no header"); const header = batch.header; const items = batch.items ?? []; + const ctx: RouteContext = { + batchId: batch.id, + source: batch.source, + sharedBy: batch.shared_by, + platform: header.platform, + header, + items, + currency: header.currency, + }; + + // The card was built minutes ago and the tap arrives now. In between, the + // email/order bridge books orders on its own schedule — so re-run the match + // rather than trusting the "no match" the card was drawn from. Creating a + // second row for an order the ledger already holds is the one outcome this + // whole slice exists to avoid. + const { match } = await matchTransaction(header, header.platform); + if (match) { + await replaceLineItems(match.transaction_id, items, { source: "intake", platform: header.platform, batchId: batch.id, metadataId: match.metadata_id }); + await updateBatch(batch.id, { kind, status: "enriched", transaction_id: match.transaction_id, items }); + return finish(ctx, kind, "enriched", enrichedCard({ ...cardInput(ctx), transactionId: match.transaction_id }), { transaction_id: match.transaction_id }); + } + + // A sibling batch of the same order — the PDF after the screenshot — may have + // been resolved since. `findDuplicateBatch` deliberately ignores unresolved + // batches, so two pending cards for one order both reach here; without this + // the second tap books the order twice. + const twin = await findDuplicateBatch(batch.id, batch.file_sha256, header); + if (twin) { + await updateBatch(batch.id, { kind, status: "duplicate", transaction_id: twin.transaction_id, capture_id: twin.capture_id }); + const card = duplicateCard({ merchant: header.merchant, total: header.total, currency: header.currency, transactionId: twin.transaction_id, captureId: twin.capture_id, batchId: twin.id }); + return finish(ctx, kind, "duplicate", card, { + transaction_id: twin.transaction_id ?? undefined, + capture_id: twin.capture_id ?? undefined, + duplicate_of: { batch_id: twin.id, transaction_id: twin.transaction_id, capture_id: twin.capture_id }, + }); + } + const ownerId = await ownerForEmail(batch.shared_by); const merchant = header.merchant ?? "Unknown merchant"; + // Not `orderDescription()`: it always prefixes "Order - ", which is right for + // a delivery receipt and wrong for the in-store till receipt this path also + // handles. Not `resolveCategory()` either — it re-derives grocery-vs-food + // from a merchant pattern, and `kind` is the answer the router already + // reached, remembered merchant and Slack tap included. const description = header.platform ? `Order - ${merchant}` : merchant; const category = kind === "grocery" ? "groceries" : "dining"; - // source/source_ref make a second press of the same card a no-op rather than a second row. - const existing = await queryRow<{ id: number }>(`SELECT id FROM transactions WHERE source = 'intake' AND source_ref = $1`, [batch.id]); + // `amount` is the figure printed on the receipt, in its own currency. No FX + // rate exists at intake, so a foreign order leaves `amount_aud` NULL rather + // than asserting a conversion — the same shape order-ingestion.ts writes. + const currency = header.currency ?? "AUD"; + const isAud = currency === "AUD"; + // source/source_ref make a second press of the same card a no-op rather than a + // second row — and ON CONFLICT, not SELECT-then-INSERT, so two presses that + // arrive together cannot both find nothing and both insert. + const inserted = await queryRow<{ id: number }>( + `INSERT INTO transactions ( + transaction_date, description, amount, amount_aud, category, payment_method, + merchant_name, merchant_normalized, transaction_type, owner_id, source, source_ref, + foreign_currency_amount, foreign_currency_code + ) VALUES ($1::date, $2, $3::numeric, $4::numeric, $5, NULL, $6, $6, 'debit', $7, 'intake', $8, $9::numeric, $10) + ON CONFLICT (source, source_ref) WHERE source IS NOT NULL AND source_ref IS NOT NULL + DO NOTHING + RETURNING id`, + [ + header.order_date, + description, + header.total, + isAud ? header.total : null, + category, + merchant, + ownerId, + batch.id, + isAud ? null : header.total, + isAud ? null : currency, + ] + ); const txnId = - existing?.id ?? - ( - await queryRow<{ id: number }>( - `INSERT INTO transactions ( - transaction_date, description, amount, amount_aud, category, payment_method, - merchant_name, merchant_normalized, transaction_type, owner_id, source, source_ref - ) VALUES ($1::date, $2, $3::numeric, $3::numeric, $4, NULL, $5, $5, 'debit', $6, 'intake', $7) - RETURNING id`, - [header.order_date, description, header.total, category, merchant, ownerId, batch.id] - ) - )!.id; + inserted?.id ?? + (await queryRow<{ id: number }>(`SELECT id FROM transactions WHERE source = 'intake' AND source_ref = $1`, [batch.id]))!.id; await replaceLineItems(txnId, items, { source: "intake", platform: header.platform, batchId: batch.id }); await updateBatch(batch.id, { kind, status: "created", transaction_id: txnId }); - const ctx: RouteContext = { batchId: batch.id, source: batch.source, sharedBy: batch.shared_by, platform: header.platform, header, items }; return finish(ctx, kind, "created", createdCard({ ...cardInput(ctx), transactionId: txnId }), { transaction_id: txnId }); } /** The card for a batch as it stands — what a second press of a resolved card gets back. */ export function resolvedCard(batch: BatchRow): { blocks: Block[]; text: string } { const h = batch.header; - const base: CardInput = { batchId: batch.id, merchant: h?.merchant ?? null, total: h?.total ?? null, orderDate: h?.order_date ?? null, platform: h?.platform ?? null, itemCount: batch.items?.length ?? 0 }; + const base: CardInput = { batchId: batch.id, merchant: h?.merchant ?? null, total: h?.total ?? null, orderDate: h?.order_date ?? null, platform: h?.platform ?? null, itemCount: batch.items?.length ?? 0, currency: h?.currency }; if (batch.status === "enriched" && batch.transaction_id) return enrichedCard({ ...base, transactionId: batch.transaction_id }); if (batch.status === "created" && batch.transaction_id) return createdCard({ ...base, transactionId: batch.transaction_id }); if (batch.status === "forwarded_to_pantry" && batch.capture_id) return forwardedCard({ ...base, captureId: batch.capture_id, enriching: !!batch.transaction_id }); - return duplicateCard({ merchant: base.merchant, total: base.total, transactionId: batch.transaction_id, captureId: batch.capture_id, batchId: batch.id }); + return duplicateCard({ merchant: base.merchant, total: base.total, currency: base.currency, transactionId: batch.transaction_id, captureId: batch.capture_id, batchId: batch.id }); } // ─── the router ───────────────────────────────────────────────────────────── @@ -438,7 +542,7 @@ export async function runIntake(input: IntakeInput, deps: IntakeDeps): Promise