diff --git a/prisma/migrations/0030_order_transaction_links/migration.sql b/prisma/migrations/0030_order_transaction_links/migration.sql new file mode 100644 index 0000000..4e9f514 --- /dev/null +++ b/prisma/migrations/0030_order_transaction_links/migration.sql @@ -0,0 +1,127 @@ +-- 0030 — order ↔ transaction, many-to-many (board 205, orders phase 2) +-- +-- Phase 1 read linkage out of `expense_metadata`, which is shaped as ONE ROW +-- PER TRANSACTION: `transaction_id` is UNIQUE and `matched_transaction_id` +-- partial-unique. Every multiplicity it expresses today is smuggled through a +-- string key — migration 0029 keys split shipments `#f` — +-- and a BNPL plan needs four rows for one order with no such trick available. +-- +-- The flagship case: order_ebay_14-11714-95953, a A$1,599 DJI drone paid in +-- four A$399.75 Afterpay legs (txns 2318, 2333, 1652, 1664). Phase 1 shows it +-- with zero linked transactions, which is correct and useless. +-- +-- KEYED ON entity_key, NOT entities.id. finance-app does not model the spine +-- and must not carry an FK across an ownership boundary that a spine +-- re-extraction can decompile. The real cascade risk is NOT transaction +-- deletion (that is handled below) but spine RE-KEYING: supersede_stale() +-- decompiles an entity when a re-extraction yields a different entity_key, and +-- a TEXT key with no FK silently orphans. Bridge-sourced links are rebuildable; +-- `manual` ones are curation and are not. Hence the orphan view at the end. + +CREATE TABLE IF NOT EXISTS order_transaction_links ( + id SERIAL PRIMARY KEY, + entity_key TEXT NOT NULL, + transaction_id INTEGER NOT NULL REFERENCES transactions(id) ON DELETE CASCADE, + leg_kind TEXT NOT NULL CHECK (leg_kind IN ('charge','shipment','instalment','refund','fee')), + -- HUMAN position only: "2 of 4". Never an id. + -- + -- The first draft stored a fact id here. Live shipment keys already carry + -- fact ids 22664 and 22711, max(extracted_facts.id) is 23,781 today, and + -- SMALLINT tops out at 32,767 — that ceiling arrives on ordinary corpus + -- growth and the insert dies with `smallint out of range`. Fact ids go in + -- source_fact_id, which is BIGINT. + leg_index SMALLINT, + leg_count SMALLINT, + source_fact_id BIGINT, + -- The LEG's own amount, signed: positive is money out, negative is a credit + -- coming back. Storing the order total on every leg would make four rows sum + -- to four times the purchase. + amount NUMERIC(12,2) NOT NULL, + currency TEXT NOT NULL DEFAULT 'AUD', + source TEXT NOT NULL CHECK (source IN ('order-bridge','instalment-matcher','manual')), + confidence TEXT NOT NULL DEFAULT 'exact' CHECK (confidence IN ('exact','derived','manual')), + evidence JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Every index needs IF NOT EXISTS, not just the table: a run that creates the +-- table and then fails during the backfill would otherwise abort on re-run +-- with "relation already exists" and need manual surgery. +-- +-- uq_otl_transaction is load-bearing. Without it two matchers can each claim +-- txn 2318 and the order reads as paid twice. It is deliberately stricter than +-- reality for one case — a single card charge covering two orders — which then +-- surfaces as a NAMED REJECTED INSERT rather than as silent duplication. +CREATE UNIQUE INDEX IF NOT EXISTS uq_otl_transaction ON order_transaction_links (transaction_id); + +-- WHAT IDENTIFIES A LEG DEPENDS ON ITS KIND, and the first version of this +-- index got it wrong in the most on-brand way available. It read +-- (entity_key, leg_kind, COALESCE(leg_index, 0)), so two shipment legs of the +-- same order — both with a NULL leg_index — collapsed to the same key and +-- ON CONFLICT DO NOTHING dropped one SILENTLY. Caught only because the +-- backfill reported 62 rows against 63 candidates. +-- +-- The row it ate was order_amazon_249-4859367-0690246's $130.00 second +-- shipment (txn 3171) — the same order named in migration 0029's comment as +-- the reason split shipments need distinct keys at all, because Amazon mails +-- and CHARGES per shipment. A uniqueness rule that cannot hold two shipments +-- is the exact defect 0029 was written to fix. +-- +-- So: an instalment leg is identified by its INDEX (1 of 4), a shipment leg by +-- its FACT (each shipment is its own extracted fact), and a whole-order charge +-- by neither — one per order, which the zeros express. +DROP INDEX IF EXISTS uq_otl_leg; +CREATE UNIQUE INDEX IF NOT EXISTS uq_otl_leg ON order_transaction_links + (entity_key, leg_kind, COALESCE(leg_index, 0), COALESCE(source_fact_id, 0)); +CREATE INDEX IF NOT EXISTS idx_otl_entity ON order_transaction_links (entity_key); + +-- Backfill the existing bridge rows so the table is not empty on day one and +-- the readers can switch over in the same change. `#f` splits into a +-- shipment leg carrying its fact id; everything else is a whole-order charge. +INSERT INTO order_transaction_links + (entity_key, transaction_id, leg_kind, leg_index, source_fact_id, amount, currency, source, confidence, evidence) +SELECT split_part(em.source_message_id, '#', 1) AS entity_key, + COALESCE(em.matched_transaction_id, em.transaction_id) AS transaction_id, + CASE WHEN em.source_message_id LIKE '%#f%' THEN 'shipment' ELSE 'charge' END, + NULL::smallint, + NULLIF(substring(em.source_message_id FROM '#f([0-9]+)$'), '')::bigint, + COALESCE(em.amount, 0), + COALESCE(em.currency, 'AUD'), + 'order-bridge', + 'exact', + jsonb_build_object('backfilled_from', 'expense_metadata', + 'source_message_id', em.source_message_id) + FROM expense_metadata em + WHERE em.source = 'order-bridge' + AND COALESCE(em.matched_transaction_id, em.transaction_id) IS NOT NULL +ON CONFLICT DO NOTHING; + +-- Spine re-keying orphans links silently. Bridge rows can be rebuilt by +-- re-running the bridge; a `manual` orphan is lost curation and is the row +-- that actually needs a human. Same idea as order_settlement_violations. +CREATE OR REPLACE VIEW order_link_orphans AS +SELECT l.id, l.entity_key, l.transaction_id, l.leg_kind, l.leg_index, + l.amount, l.currency, l.source, l.confidence, l.created_at + FROM order_transaction_links l + WHERE NOT EXISTS (SELECT 1 FROM entities e WHERE e.entity_key = l.entity_key); + +-- Two stores that can disagree will disagree. The bridge now writes both in +-- ONE transaction, so a half-written pair should be impossible — this view +-- exists to prove that rather than to assume it, and it must stay empty. +-- +-- The plan's suggested guard (widen the bridge's `NOT EXISTS expense_metadata` +-- to "neither store has it") was NOT taken: the expense_metadata INSERT has no +-- ON CONFLICT, so re-attempting an order that already has a receipt row would +-- duplicate it. Detecting drift is cheap; re-attempting a non-idempotent write +-- to fix a state that cannot occur is not. +CREATE OR REPLACE VIEW order_link_drift AS +SELECT em.source_message_id, + COALESCE(em.matched_transaction_id, em.transaction_id) AS transaction_id, + em.amount, em.currency, em.reconciled_at + FROM expense_metadata em + WHERE em.source = 'order-bridge' + AND COALESCE(em.matched_transaction_id, em.transaction_id) IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM order_transaction_links l + WHERE l.transaction_id = COALESCE(em.matched_transaction_id, em.transaction_id) + ); diff --git a/src/app/api/transactions/[id]/order/route.ts b/src/app/api/transactions/[id]/order/route.ts index 844815f..757b1f5 100644 --- a/src/app/api/transactions/[id]/order/route.ts +++ b/src/app/api/transactions/[id]/order/route.ts @@ -38,7 +38,30 @@ export async function GET( [Number(id)] ); + // Phase 2 (board 205). A BNPL leg has NO expense_metadata row of its own — + // the plan is four transactions against one order — so without this the four + // Afterpay debits behind the A$1,599 DJI drone stay four bare "Afterpay + // $399.75" rows naming nothing. That is the surface where this was noticed. + const link = await queryRow<{ + entity_key: string; leg_kind: string; leg_index: number | null; + leg_count: number | null; canonical_name: string | null; + platform: string | null; order_total: string | null; currency: string | null; + }>( + `SELECT l.entity_key, l.leg_kind, l.leg_index::int, l.leg_count::int, + e.canonical_name, o.platform, o.order_total, o.currency + FROM order_transaction_links l + LEFT JOIN entities e ON e.entity_key = l.entity_key + LEFT JOIN entity_orders o ON o.entity_id = e.id + WHERE l.transaction_id = $1 + LIMIT 1`, + [Number(id)] + ); + // Not an order — most transactions aren't. Null, not 404: the caller is // asking "is there a receipt behind this?", and "no" is a normal answer. - return NextResponse.json(row ?? null); + if (!row && !link) return NextResponse.json(null); + + // expense_metadata carries the itemised receipt and stays primary; the link + // adds what it cannot express — which order this is a leg OF, and which leg. + return NextResponse.json({ ...(row ?? {}), order_link: link ?? null }); } diff --git a/src/app/orders/[entityKey]/page.tsx b/src/app/orders/[entityKey]/page.tsx index a1c7b69..ccec24f 100644 --- a/src/app/orders/[entityKey]/page.tsx +++ b/src/app/orders/[entityKey]/page.tsx @@ -226,6 +226,15 @@ export default function OrderDetailPage({ params }: { params: Promise<{ entityKe {t.description} + {/* Board 205. Four identical "Afterpay" rows are + indistinguishable without their position, and + indistinguishable rows are what made this order + look like four purchases. */} + {t.leg_kind === "instalment" && t.leg_index && t.leg_count + ? `Instalment ${t.leg_index} of ${t.leg_count} · ` + : t.leg_kind === "shipment" + ? "Shipment · " + : ""} {dateFmt.format(new Date(t.transaction_date))} · txn {t.transaction_id} diff --git a/src/app/transactions/page.tsx b/src/app/transactions/page.tsx index 227f0ea..acf02ce 100644 --- a/src/app/transactions/page.tsx +++ b/src/app/transactions/page.tsx @@ -1018,6 +1018,17 @@ function TransactionsContent() { {t.notes ? (

{t.notes}

+ ) : t.order_leg_kind === "instalment" && t.order_name ? ( + // Board 205. Four rows reading "Afterpay $399.75" name + // nothing — this is the surface where that was noticed. + // The order is what tells them apart, and "2 of 4" is + // what stops the same purchase reading as four. +

+ {t.order_leg_index && t.order_leg_count + ? `${t.order_leg_index} of ${t.order_leg_count} · ` + : ""} + {t.order_name} +

) : t.order_platform === "uber" && routeSummary(t.order_route) && ( // Five rows all reading "Order - Uber Trip" are // indistinguishable. Where the trip went is what tells diff --git a/src/components/order-details.tsx b/src/components/order-details.tsx index 9e41890..5d300a3 100644 --- a/src/components/order-details.tsx +++ b/src/components/order-details.tsx @@ -72,7 +72,12 @@ export function OrderDetails({ const { data: review } = useOrderReview(transactionId); const [reviewer, setReviewer] = useState(OWNER_PARTICIPANT_ID); - if (isLoading || !receipt) return null; + // `platform` is the marker of a real receipt. Since phase 2 the endpoint also + // answers with `{ order_link }` alone for a transaction that is merely LINKED + // to an order (a BNPL leg has no receipt of its own), and that object is + // truthy — without this check the panel would render a receipt shell with no + // merchant, no items and no total. + if (isLoading || !receipt || !receipt.platform) return null; const cur = receipt.currency ?? currency ?? "AUD"; const fmt = (n: number) => (cur === "AUD" ? `$${n.toFixed(2)}` : `${cur} ${n.toFixed(2)}`); diff --git a/src/lib/order-feed.ts b/src/lib/order-feed.ts index 2a6f972..c6c05ab 100644 --- a/src/lib/order-feed.ts +++ b/src/lib/order-feed.ts @@ -205,18 +205,41 @@ function buildWhere(filters: OrderFilters) { } /** - * Ledger linkage. Phase 1 reads expense_metadata; phase 2 swaps this lateral - * AND the detail query AND /api/transactions/[id]/order together — moving only - * one of the three leaves the list contradicting the detail page. + * Ledger linkage — phase 2 (board 205). + * + * `order_transaction_links` is the source of truth: it is many-to-many, so a + * BNPL plan is four rows against one order. `expense_metadata` remains as a + * FALLBACK because `jobs/order_transaction_bridge.py` still writes there on + * every hourly tick; until a bridge row is mirrored into links, dropping the + * fallback would make freshly bridged orders read as unpaid. + * + * UNION, not UNION ALL, on transaction_id: after the 0030 backfill the same + * charge is legitimately present in both stores, and counting it twice would + * show "2 charges" on a single-payment order. + * + * All four readers move together — this lateral, the detail query below, + * /api/transactions/[id]/order, and the order_ctx lateral in queries.ts. + * Moving fewer is not a smaller change, it is an inconsistent one: once the + * matcher inserts four links for order_ebay_14-11714-95953 that order still + * has no expense_metadata row, so the detail page would show four instalments + * while the list showed txn_count = 0 and the has_transaction facet put it on + * the wrong side of both filters. */ const LINK_LATERAL = ` LEFT JOIN LATERAL ( - SELECT count(*)::int AS txn_count, - min(em.matched_transaction_id) AS first_txn_id - FROM expense_metadata em - WHERE em.source = 'order-bridge' - AND (em.source_message_id = f.entity_key - OR em.source_message_id LIKE f.entity_key || '#f%') + SELECT count(*)::int AS txn_count, min(txn_id) AS first_txn_id + FROM ( + SELECT l.transaction_id AS txn_id + FROM order_transaction_links l + WHERE l.entity_key = f.entity_key + UNION + SELECT COALESCE(em.matched_transaction_id, em.transaction_id) + FROM expense_metadata em + WHERE em.source = 'order-bridge' + AND (em.source_message_id = f.entity_key + OR em.source_message_id LIKE f.entity_key || '#f%') + AND COALESCE(em.matched_transaction_id, em.transaction_id) IS NOT NULL + ) both_stores ) link ON true`; const FROM_CLAUSE = ` @@ -360,6 +383,12 @@ export interface OrderLinkedTxn { description: string; amount: string; source_message_id: string; + /** 'charge' | 'shipment' | 'instalment' | 'refund' | 'fee'. A plan's legs are + * four 'instalment' rows; a whole-order card charge is one 'charge'. */ + leg_kind: string; + /** Human position, "2 of 4" — populated for instalments only. */ + leg_index: number | null; + leg_count: number | null; } export interface OrderDetail { @@ -490,14 +519,35 @@ export async function getOrderDetail(entityKey: string): Promise( - `SELECT t.id::int AS transaction_id, t.transaction_date, t.description, - t.amount, em.source_message_id - FROM expense_metadata em - JOIN transactions t - ON t.id = COALESCE(em.matched_transaction_id, em.transaction_id) - WHERE em.source = 'order-bridge' - AND (em.source_message_id = $1 OR em.source_message_id LIKE $1 || '#f%') - ORDER BY t.transaction_date`, + // Links first, expense_metadata as fallback, deduped on transaction_id — + // see LINK_LATERAL above for why both stores are read. DISTINCT ON keeps + // the LINK row when a transaction is in both, because only that row knows + // whether the payment was one charge or leg 3 of 4. + // The DISTINCT ON must be ordered by t.id, so the chronological sort the + // page needs goes on the OUTER query — legs read as "1, 2, 3, 4" only if + // they come back by date. + `SELECT * FROM ( + SELECT DISTINCT ON (t.id) + t.id::int AS transaction_id, t.transaction_date, t.description, + t.amount, both_stores.source_message_id, + both_stores.leg_kind, both_stores.leg_index, both_stores.leg_count + FROM ( + SELECT l.transaction_id, l.leg_kind, l.leg_index::int, l.leg_count::int, + l.entity_key AS source_message_id, 0 AS pref + FROM order_transaction_links l + WHERE l.entity_key = $1 + UNION ALL + SELECT COALESCE(em.matched_transaction_id, em.transaction_id), + 'charge', NULL::int, NULL::int, em.source_message_id, 1 + FROM expense_metadata em + WHERE em.source = 'order-bridge' + AND (em.source_message_id = $1 OR em.source_message_id LIKE $1 || '#f%') + AND COALESCE(em.matched_transaction_id, em.transaction_id) IS NOT NULL + ) both_stores + JOIN transactions t ON t.id = both_stores.transaction_id + ORDER BY t.id, both_stores.pref + ) deduped + ORDER BY transaction_date, leg_index NULLS FIRST`, [entityKey] ); diff --git a/src/lib/queries.ts b/src/lib/queries.ts index 52a67af..ac98c4e 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -37,7 +37,16 @@ export interface TransactionRow { payment_method: string | null; /** Uber pick-up/drop-off, when this row came from an order receipt. */ order_route: RoutePointRow[] | null; + /** Receipt platform ONLY — it gates the disclosure arrow, so it must stay + * true to "there is a receipt behind this row". */ order_platform: "doordash" | "ubereats" | "uber" | null; + /** Phase 2 (board 205) — set when this transaction is linked to an order. + * 'instalment' means it is one leg of a plan; leg_index/leg_count carry + * "2 of 4", which is what stops one purchase reading as four. */ + order_leg_kind: string | null; + order_leg_index: number | null; + order_leg_count: number | null; + order_name: string | null; // override fields category_override: string | null; merchant_override: string | null; @@ -314,7 +323,17 @@ export async function getTransactions(ownerId: number, filters: TransactionFilte txn_tags.tags, txn_splits.splits, order_ctx.route as order_route, - order_ctx.platform as order_platform + -- Deliberately NOT COALESCEd with the link's platform. order_platform + -- gates the receipt disclosure arrow on /transactions, and a BNPL leg + -- has no receipt behind it — filling this in put an arrow on four + -- Afterpay rows that expand to nothing, which is the exact promise the + -- arrow exists to avoid making. The leg fields below carry the sub-line + -- instead. + order_ctx.platform as order_platform, + order_link.leg_kind as order_leg_kind, + order_link.leg_index as order_leg_index, + order_link.leg_count as order_leg_count, + order_link.canonical_name as order_name FROM transactions t LEFT JOIN transaction_overrides o ON o.transaction_id = t.id LEFT JOIN statements s ON s.id = t.statement_id @@ -323,12 +342,27 @@ export async function getTransactions(ownerId: number, filters: TransactionFilte -- only thing that tells them apart, and it was already stored. -- Both directions, because a card-settled order has no transaction of its -- own and points at the statement line instead (I5). + -- Phase 2 (board 205): a BNPL leg has no expense_metadata row of its own, + -- so COALESCE in the link's platform. Without it the four Afterpay debits + -- behind the A$1,599 drone keep an empty sub-line while their order sits + -- one join away. The route column only ever exists on the receipt side. + -- (No backticks in here: this SQL lives in a TS template literal and a + -- backtick ends the string — TS1005 on a line that looks like a comment.) LEFT JOIN LATERAL ( SELECT em.route, em.platform FROM expense_metadata em WHERE em.transaction_id = t.id OR em.matched_transaction_id = t.id LIMIT 1 ) order_ctx ON true + LEFT JOIN LATERAL ( + SELECT o.platform, l.leg_kind, l.leg_index::int AS leg_index, + l.leg_count::int AS leg_count, e.canonical_name + FROM order_transaction_links l + LEFT JOIN entities e ON e.entity_key = l.entity_key + LEFT JOIN entity_orders o ON o.entity_id = e.id + WHERE l.transaction_id = t.id + LIMIT 1 + ) order_link ON true LEFT JOIN participants p ON p.id = COALESCE(t.owner_id, s.owner_id) LEFT JOIN transactions src ON src.reconciled_with_id = t.id AND src.statement_id IS NULL LEFT JOIN trips tr ON tr.id = o.trip_id