fix(orders): two defects the backfill exposed that tests could not
ci / lint-test (push) Failing after 1m26s

Both were found by looking at the data after the live backfill, not by the
suite — 105 tests were green while 85 rows were invisible and 4 were double
counted.

owner_id was NULL on every ingested order. Analytics scope on
COALESCE(t.owner_id, s.owner_id), and an ingested order carries no statement,
so the coalesce resolved to NULL and matched no owner. The rows existed in
`transactions` and appeared in no view in the app. Ingestion now sets
DEFAULT_OWNER_ID, and a regression test asserts the row survives the same
COALESCE scoping the UI uses.

[Family] orders are card-settled, not credits-funded. Their receipts name the
payer ("Payments Siddharth LKR 3,783.20") and no instrument, which an earlier
version read as credits. The card statement carries all four of them (CBA
...3893, exact foreign_currency_amount matches), so creating a transaction
duplicated spend already recorded — the double-count I5 exists to prevent.
They now record provenance only; the statement line is the transaction and is
what carries the `family` tag that keeps them out of budgets.

Production data corrected separately by
smarthome:docker/scripts/fix-order-backfill-2026-07-27.sql.

Also: reconciliation tests no longer assert global row counts.
reconcilePendingOrders() scans every pending row, so leftovers from other
files moved the totals — the source of an intermittent failure that only
appeared on the first run after a source edit.
This commit is contained in:
2026-07-27 10:41:10 +10:00
parent 5db42f086f
commit ae0c34fce7
4 changed files with 78 additions and 56 deletions
@@ -9,7 +9,6 @@ import {
reconcilePendingOrders,
parseOrderAmendment,
applyOrderAmendment,
OrderParseError,
NotAReceiptError,
type MessageMeta,
} from "../../lib/order-ingestion";
@@ -181,14 +180,22 @@ describe("Order ingestion — invariants", () => {
).rejects.toThrow();
});
it("I11: a [Family] order is imported, tagged, and excluded from spend", async () => {
it("I11: a [Family] order records provenance and creates no transaction", async () => {
// Requirement was "import them but tag so they're excluded from budgets".
// Correct mechanism: the CARD statement line is the transaction and carries
// the family tag. Creating a second, credits-flavoured row duplicated it.
const p = parseOrderHTML(
html("ue-04"),
meta({ subject: "[Family] Your Tuesday evening order with Uber Eats", sender: "uber.com", receivedAt: "2026-07-07T10:08:00Z" })
);
const res = await processOrderIngestion(p);
expect(res.transactionId).not.toBeNull();
expect(res.flags).toContain("family_payment_assumed_credits");
expect(res.transactionId).toBeNull();
expect(res.flags).toContain("family_card_settled_no_transaction");
const meta_ = await queryRow<{ currency: string }>(
`SELECT currency FROM expense_metadata WHERE id = $1`, [res.metadataId]
);
expect(meta_!.currency).toBe("LKR");
});
it("a foreign-currency order records the original amount and code", async () => {
@@ -215,6 +222,12 @@ describe("Order ingestion — invariants", () => {
expect(res.transactionId).toBeNull();
expect(res.flags).toContain("awaiting_card_statement");
// Repeat runs would otherwise accumulate identical candidate charges.
await queryRaw(
`DELETE FROM transactions WHERE description = 'DD *DOORDASH WOOLWORTHS MELBOURNE AUS'`
);
await queryRaw(`DELETE FROM statements WHERE filename = 'test-westpac-2026-03.pdf'`);
// Statement arrives: card 8032 took 40.93 of the 60.93 order.
const st = await queryRow<{ id: number }>(
`INSERT INTO statements (bank_name, account_number, filename)
@@ -227,8 +240,11 @@ describe("Order ingestion — invariants", () => {
);
const out = await reconcilePendingOrders();
expect(out.resolved).toBeGreaterThanOrEqual(1);
expect(out.created).toBeGreaterThanOrEqual(1);
// Deliberately not asserting global counts: reconcilePendingOrders() scans
// every pending row in the database, so another test's leftovers change the
// totals. Assert on THIS order's outcome instead — that is what the test is
// actually about, and it does not depend on what else is in the table.
expect(out.examined).toBeGreaterThanOrEqual(1);
const credits = await queryRow<{ amount: string }>(
`SELECT t.amount::text FROM transactions t
@@ -240,9 +256,12 @@ describe("Order ingestion — invariants", () => {
});
it("reconciliation is idempotent — a second pass creates nothing", async () => {
const before = await queryRow<{ c: string }>(`SELECT count(*)::text c FROM transactions`);
// A first pass has already run above; a second must add nothing. Scoped to
// 'Order - %' rows so unrelated fixtures cannot move the number.
const q = `SELECT count(*)::text c FROM transactions WHERE description LIKE 'Order - %'`;
const before = await queryRow<{ c: string }>(q);
const out = await reconcilePendingOrders();
const after = await queryRow<{ c: string }>(`SELECT count(*)::text c FROM transactions`);
const after = await queryRow<{ c: string }>(q);
expect(out.created).toBe(0);
expect(after!.c).toBe(before!.c);
});
@@ -320,39 +339,43 @@ describe("Refund amendments", () => {
});
});
describe("[Family] orders import rather than park", () => {
it("records a family order as credits and tags it", async () => {
describe("[Family] orders are card-settled, not credits", () => {
it("creates provenance but NO transaction — the statement line is the transaction", async () => {
// Regression: these were assumed credits-funded because the receipt names
// the payer and no instrument. The card statement carries all four (CBA
// ...3893, exact LKR matches), so creating a transaction double-counted
// spend already recorded.
const p = parseOrderHTML(
html("ue-04"),
meta({ subject: "[Family] Your Tuesday evening order with Uber Eats", sender: "uber.com", receivedAt: "2026-07-07T10:08:00Z" })
);
expect(p.flags).toContain("family_payment_assumed_credits");
expect(p.flags).toContain("family_card_settled_no_transaction");
expect(p.payment.credits_amount).toBeNull();
const res = await processOrderIngestion(p);
expect(res.transactionId).not.toBeNull();
expect(res.transactionId).toBeNull();
expect(res.metadataId).not.toBeNull();
});
});
const tag = await queryRow<{ name: string }>(
`SELECT tg.name FROM transaction_tags tt JOIN tags tg ON tg.id = tt.tag_id
WHERE tt.transaction_id = $1`,
describe("owner scoping", () => {
it("sets owner_id so the row is visible to the app", async () => {
// Regression: analytics scope on COALESCE(t.owner_id, s.owner_id). An
// ingested order has no statement, so a NULL owner_id made all 85
// backfilled rows invisible in every view while sitting in the table.
const p = parseOrderHTML(html("dd-01"), meta({ messageId: `owner-${Date.now()}` }));
const res = await processOrderIngestion(p);
const row = await queryRow<{ owner_id: number | null }>(
`SELECT owner_id FROM transactions WHERE id = $1`,
[res.transactionId]
);
expect(tag!.name).toBe("family");
expect(row!.owner_id).not.toBeNull();
// LKR is preserved, and amount_aud stays NULL — no FX rate is available.
const txn = await queryRow<{ foreign_currency_code: string; amount_aud: string | null }>(
`SELECT foreign_currency_code, amount_aud::text FROM transactions WHERE id = $1`,
[res.transactionId]
);
expect(txn!.foreign_currency_code).toBe("LKR");
expect(txn!.amount_aud).toBeNull();
// And it must not reach spend.
const visible = await queryRaw(
`SELECT t.id FROM transactions t
LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
WHERE t.id = $1 AND (${EXCLUDE_NON_SPEND})`,
[res.transactionId]
`SELECT t.id FROM transactions t LEFT JOIN statements s ON s.id = t.statement_id
WHERE t.id = $1 AND COALESCE(t.owner_id, s.owner_id) = $2`,
[res.transactionId, row!.owner_id]
);
expect(visible).toHaveLength(0);
expect(visible).toHaveLength(1);
});
});
@@ -5,7 +5,6 @@ import {
parseOrderHTML,
validateOrderTotals,
resolveCategory,
OrderParseError,
NotAReceiptError,
type MessageMeta,
type ParsedOrder,