wallet: lock the ingest, pair on time and app, skip notices, ECB fallback

Four defects found reviewing a month of overseas wallet rows: - Race: n8n posts a poll's items concurrently and both halves of a tap usually share a poll, so each ran findPairedRow before the other inserted. 11 taps became 22 rows (A$3,841.40), identical created_at on every pair. One advisory lock now covers re-check -> pair -> adopt -> insert, on the transaction's own connection (a second pooled one per request deadlocks a burst). Mutation-tested: without the lock the concurrent test fails 3/3. - Pair window: amount within +/-1 DAY folded the UNICEF $50 donation and an unrelated $50 Amex charge 17h later into one row and renamed it. A pair is now minutes apart AND from two different apps. Needs when/where a notification arrived: migration 0039 adds expense_metadata.notified_at / notif_app, backfilled from capture.wallet_events (144 rows). A merged half now records wallet_evt:<id>, so a replay cannot merge it twice. - Not purchases: a CommBank overdraft notice became a $21.21 purchase named "To avoid fees and interest," (the Westpac fixed-width parser ran on a CommBank body). Account notices and transit-card balance drawdowns are skipped; the descriptor parser only runs on Westpac. - FX: with no ledger history (Hong Kong) every tap stored A$0.00. The ECB rate via Frankfurter is the fallback, flagged rate_ecb; the ledger median stays first because it carries the card's FX fee. Also: dates in Australia/Melbourne like the statements (UTC slicing filed 28 of 134 rows a day early), and test:setup now actually rebuilds personal_test -- replaying the dump over it never applied a migration. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-03 16:55:07 +10:00
parent 22fde30b26
commit a3e4e1102a
8 changed files with 450 additions and 146 deletions
@@ -77,6 +77,10 @@ beforeAll(async () => {
 });

 beforeEach(async () => {
+  // No network in tests. The ECB fallback treats any fetch failure as "no
+  // rate", which is what every pre-existing currency test assumes; the one test
+  // about the fallback stubs a rate of its own.
+  vi.stubGlobal("fetch", async () => { throw new Error("no network in tests"); });
  await resetDB(pool);
  await pool.query("DELETE FROM expense_metadata");
  ({ ownerId } = await seedParticipants(pool));
@@ -147,7 +151,128 @@ describe("the notification pair", () => {
  });
 });

+describe("the notification pair is recognised by time and app, not amount alone", () => {
+  it("makes ONE row when both halves are posted at the same moment", async () => {
+    // n8n posts a poll's items concurrently, and both halves of a tap usually
+    // land in the same poll. Before the lock, each ran its pair lookup before
+    // the other inserted: 11 taps became 22 rows, identical created_at on each.
+    await Promise.all([ingest(req(breadworldBank)), ingest(req(breadworldWallet))]);
+    expect(await txns()).toHaveLength(1);
+    const [m] = await metas();
+    expect(m.card_last4).toBe("8032");
+  });
+
+  it("keeps two same-amount charges hours apart as two purchases", async () => {
+    // Event 275 ($50 at A MEDIUM) merged into the UNICEF $50 donation from 17
+    // hours earlier and renamed it, because the old window was +/-1 day.
+    await ingest(req({
+      ...amexRewardPay, event_id: 267, amount: 50, amount_text: "A$50.00", merchant: "UNICEF",
+      notif_body: "You have made a A$50.00 charge on your American Express® Card ending in 01000 at UNICEF.",
+      occurred_at: "2026-10-01T11:02:40+10:00",
+    }));
+    await ingest(req({
+      ...amexRewardPay, event_id: 275, amount: 50, amount_text: "$50.00", merchant: "A MEDIUM",
+      notif_body: "You have made a $50.00 charge on your American Express® Card ending in 01000 at A MEDIUM.",
+      occurred_at: "2026-10-02T04:06:49+10:00",
+    }));
+    const rows = await txns();
+    expect(rows).toHaveLength(2);
+    expect(rows.map((r) => r.merchant_normalized).sort()).toEqual(["A MEDIUM", "UNICEF"]);
+  });
+
+  it("never pairs two notifications from the SAME app, however close", async () => {
+    // A pair is the issuer half plus the Wallet half. Two from one app are two
+    // charges -- two $20 coffees bought together, say.
+    await ingest(req(breadworldBank));
+    await ingest(req({ ...breadworldBank, event_id: 640, occurred_at: "2026-08-13T08:01:40+10:00" }));
+    expect(await txns()).toHaveLength(2);
+  });
+
+  it("records a merged half, so a replay cannot merge it again", async () => {
+    await ingest(req(breadworldBank));
+    await ingest(req(breadworldWallet));
+    const again = await (await ingest(req(breadworldWallet))).json();
+    expect(again.skipped).toBe("already_ingested");
+    const [m] = await metas();
+    expect(m.flags).toContain("wallet_evt:65");
+  });
+});
+
+describe("notifications that name an amount but are not purchases", () => {
+  it("skips an overdraft notice", async () => {
+    // Event 235 became a $21.21 purchase named "To avoid fees and interest,".
+    const body = await (await ingest(req({
+      event_id: 235, kind: "transaction", direction: "debit", amount: 21.21,
+      amount_text: "$21.21", currency: "$?", merchant: null, notif_app: "CommBank",
+      notif_body: "Your Smart Access 1910 is overdrawn by $21.21. To avoid fees and interest, bring your account to positive by midnight 21 Sep (Syd/Melb time).",
+      occurred_at: "2026-09-22T09:00:18+10:00",
+    }))).json();
+    expect(body.skipped).toBe("account_notice");
+    expect(await txns()).toHaveLength(0);
+  });
+
+  it("skips a transit-card fare, because the card's load is the spend", async () => {
+    // Event 200: the Octopus load is on the bank statement; each fare draws it
+    // down. 11 of 13 fares were already not_transaction; this makes the other 2 agree.
+    const body = await (await ingest(req({
+      event_id: 200, kind: "transaction", direction: "debit", amount: 105,
+      amount_text: "HKD 105.00", currency: "HKD", merchant: "Octopus", notif_app: "Wallet",
+      notif_title: "Octopus", notif_subtitle: "HKD 105.00 for Train",
+      notif_body: "Your new balance is -HKD 12.60.",
+      occurred_at: "2026-09-14T02:00:37+10:00",
+    }))).json();
+    expect(body.skipped).toBe("stored_value_drawdown");
+    expect(await txns()).toHaveLength(0);
+  });
+
+  it("does not read a Westpac descriptor out of another bank's message", async () => {
+    await ingest(req({
+      event_id: 2350, kind: "transaction", direction: "debit", amount: 9.5,
+      amount_text: "$9.50", currency: "$?", merchant: null, notif_app: "CommBank",
+      notif_body: "Payment of $9.50 made. Thanks for banking with us.",
+      occurred_at: "2026-09-22T09:00:18+10:00",
+    }));
+    const [t] = await txns();
+    expect(t.merchant_normalized).toBe("Card purchase");
+  });
+});
+
+describe("dating", () => {
+  it("dates a tap in Melbourne local time, as the statements do", async () => {
+    // 21:30 UTC on 30 Sep is 07:30 on 1 Oct in Melbourne. Slicing the ISO
+    // string filed it a day early.
+    await ingest(req({ ...breadworldBank, occurred_at: "2026-09-30T21:30:00Z" }));
+    const { rows } = await pool.query(
+      `SELECT transaction_date::text AS d FROM transactions WHERE source = 'wallet'`
+    );
+    expect(rows[0].d).toBe("2026-10-01");
+  });
+});
+
 describe("currency", () => {
+  it("falls back to the ECB rate when the ledger has no history", async () => {
+    // Hong Kong: no HKD statement row existed, so every tap stored A$0.00.
+    const calls: string[] = [];
+    vi.stubGlobal("fetch", async (url: string) => {
+      calls.push(String(url));
+      return { ok: true, json: async () => ({ rates: { AUD: 0.19 } }) };
+    });
+    await ingest(req({
+      event_id: 167, kind: "transaction", direction: "debit", amount: 310.1,
+      amount_text: "HKD 310.10", currency: "HKD", merchant: "Wonder Taxi", notif_app: "Wallet",
+      notif_subtitle: "Wonder Taxi, Tsim Sha Tsui East, Kowloon",
+      occurred_at: "2026-09-10T08:54:00+10:00",
+    }));
+    const [t] = await txns();
+    expect(Number(t.amount)).toBeCloseTo(58.92, 2);
+    expect(Number(t.foreign_currency_amount)).toBe(310.1);
+    expect(calls[0]).toContain("/2026-09-10?from=HKD&to=AUD");
+    const [m] = await metas();
+    // Flagged as the mid-rate, not the bank's: it lacks the card's FX fee.
+    expect(m.flags).toEqual(expect.arrayContaining(["aud_provisional", "rate_ecb"]));
+    expect(m.flags).not.toContain("no_rate_reference");
+  });
+
  it("records AUD when Amex states it explicitly", async () => {
    await ingest(req(amexRewardPay));
    const [t] = await txns();