From 8d1875225f2bedee823b72341b8407e5946b4d50 Mon Sep 17 00:00:00 2001 From: siddharthd Date: Sat, 22 Aug 2026 16:24:28 +1000 Subject: [PATCH] wallet: a transaction for every tap (board #252, ING-11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 3, and the last app-side piece. POST /api/wallet/ingest takes an interpreted notification and writes a statementless card transaction plus one expense_metadata row. Sibling to /api/orders/ingest and /api/receipts/ingest, same shared-secret shape, its own token so the wallet credential rotates without touching the order flow. Nothing here knows about statements, and that is the point: statementless + payment_method 'card' is exactly the reconcile queue's predicate, so a wallet row enters it unaided and the auto-matcher confirms it later. THE PAIR IS MERGED, NEVER DEDUPED. One tap emits two notifications ~2-8s apart: the issuer app has the card tail, Apple Wallet has the merchant and the suburb, and neither alone is complete. Matched on amount and date because the two share no identifier. My first version COALESCEd every field, which made the result depend on arrival order — the issuer half usually lands first, so the truncated all-caps "WYNDHAM VAL AU" beat the subtitle's "Wyndham Vale, VIC". A subtitle-derived place now overwrites; a descriptor-derived one only fills a gap. The test that caught this asserts both arrival orders. CURRENCY IS NEVER ASSUMED. Only Amex states one ("A$5,103.49"); Westpac, CommBank and Apple Wallet write a bare "$", which is 14 of 21 captured rows. Recorded as unknown and flagged, not assumed AUD — harmless at home, a 23x error on a THB tap abroad. When a currency IS stated and is not AUD, the NATIVE figure goes to foreign_currency_amount, because the AUD amount is the bank's own conversion and a notification cannot predict it, while the native pair is exact on both sides and is what the statement will match on. transactions.amount is NOT NULL so it takes a provisional conversion from the ledger-derived median rate, flagged aud_provisional, overwritten at reconcile. No rate history means no_rate_reference rather than an invented number. wallet-descriptor.ts moves the positional parsing out of the model, which handled ONE fixed-width format three ways on the same day's data: kept the country code on "WYNDHAM VAL AU", stripped it on "Sydney" and "Perth", and swallowed "SAN FRANCISOUS" whole — that being "SAN FRANCISO" truncated to the 12-char city field plus "US", on the only foreign merchant in the entire capture window. Every unit-test input is a real captured string. 14 unit + 12 integration tests. npm run test:all green (196 + 304), lint clean. --- .../integration/wallet-ingest.test.ts | 234 ++++++++++++++++++ src/__tests__/unit/wallet-descriptor.test.ts | 109 ++++++++ src/app/api/wallet/ingest/route.ts | 49 ++++ src/lib/wallet-descriptor.ts | 116 +++++++++ src/lib/wallet-ingestion.ts | 228 +++++++++++++++++ 5 files changed, 736 insertions(+) create mode 100644 src/__tests__/integration/wallet-ingest.test.ts create mode 100644 src/__tests__/unit/wallet-descriptor.test.ts create mode 100644 src/app/api/wallet/ingest/route.ts create mode 100644 src/lib/wallet-descriptor.ts create mode 100644 src/lib/wallet-ingestion.ts diff --git a/src/__tests__/integration/wallet-ingest.test.ts b/src/__tests__/integration/wallet-ingest.test.ts new file mode 100644 index 0000000..42e5dac --- /dev/null +++ b/src/__tests__/integration/wallet-ingest.test.ts @@ -0,0 +1,234 @@ +import { describe, it, expect, beforeAll, beforeEach, vi } from "vitest"; +import { Pool } from "pg"; +import { createPool, resetDB, seedParticipants } from "./helpers"; + +/** + * Wallet notifications becoming transactions. + * + * Every fixture is a real captured event from capture.wallet_events + * (2026-08-13 .. 08-22). The rules being tested are the ones the design turned + * on, so each has a reason rather than being a shape check: + * + * - a transaction for EVERY tap, because overseas is where live spend matters + * - the pair is MERGED, never deduped: the issuer half has the card tail, the + * Wallet half has the merchant and suburb, neither alone is complete + * - the NATIVE amount is what gets stored for a foreign charge, because the + * AUD figure is the bank's own conversion and cannot be predicted + * - a bare "$" is recorded as unknown, never assumed AUD + */ + +const TOKEN = "test-wallet-token"; +let pool: Pool; +let ownerId: number; +let ingest: typeof import("../../app/api/wallet/ingest/route").POST; + +const req = (body: unknown) => + ({ headers: { get: (h: string) => (h === "x-ingest-token" ? TOKEN : null) }, json: async () => body }) as never; + +const noAuth = (body: unknown) => + ({ headers: { get: () => null }, json: async () => body }) as never; + +// --- real captured events ------------------------------------------------- +const breadworldBank = { + event_id: 64, kind: "transaction", direction: "debit", amount: 13.5, + amount_text: "$13.50", currency: "$?", merchant: "Breadworld Bakery Cafe", + card_tail: "8032", issuer: "Mastercard", notif_app: "Westpac", + notif_body: "$13.50 paid from Mastercard ending...8032. Breadworld Bakery Cafe WYNDHAM VAL AU", + occurred_at: "2026-08-13T08:01:28+10:00", +}; +const breadworldWallet = { + event_id: 65, kind: "transaction", direction: "unknown", amount: 13.5, + amount_text: "$13.50", currency: "$?", merchant: "Breadworld Bakery Cafe", + notif_app: "Wallet", notif_subtitle: "Breadworld Bakery Cafe, Wyndham Vale, VIC", + occurred_at: "2026-08-13T08:01:35+10:00", +}; +const amexRewardPay = { + event_id: 87, kind: "transaction", direction: "debit", amount: 5103.49, + amount_text: "A$5,103.49", currency: "AUD", merchant: "REWARDPAY", + card_tail: "01000", issuer: "American Express", notif_app: "Amex AU", + notif_body: "You have made a A$5,103.49 charge on your American Express® Card ending in 01000 at REWARDPAY.", + occurred_at: "2026-08-20T11:54:43+10:00", +}; +const ebayOrderPlaced = { + event_id: 66, kind: "not_transaction", direction: "unknown", amount: null, + notif_app: "Wallet", notif_body: "eBay has received your order.", + occurred_at: "2026-08-13T09:45:55+10:00", +}; + +/** Only the rows this lane wrote -- some tests seed statement history for the + * rate reference, and those must not be mistaken for wallet output. */ +const txns = async () => + (await pool.query( + `SELECT id, amount, amount_aud, merchant_normalized, location, source, source_ref, + foreign_currency_amount, foreign_currency_code, payment_method, statement_id + FROM transactions WHERE source = 'wallet' ORDER BY id` + )).rows; + +const metas = async () => + (await pool.query(`SELECT * FROM expense_metadata ORDER BY id`)).rows; + +beforeAll(async () => { + process.env.WALLET_INGEST_TOKEN = TOKEN; + pool = createPool(); + vi.doMock("@/lib/auth", () => ({ + getCurrentUser: async () => ({ id: ownerId, name: "Alice", email: "alice@example.com" }), + })); + ({ POST: ingest } = await import("../../app/api/wallet/ingest/route")); +}); + +beforeEach(async () => { + await resetDB(pool); + await pool.query("DELETE FROM expense_metadata"); + ({ ownerId } = await seedParticipants(pool)); +}); + +describe("auth", () => { + it("fails closed without the token", async () => { + expect((await ingest(noAuth(breadworldBank))).status).toBe(401); + expect(await txns()).toHaveLength(0); + }); +}); + +describe("what reaches the ledger", () => { + it("creates a statementless card transaction for a tap", async () => { + const body = await (await ingest(req(breadworldBank))).json(); + expect(body.transactionId).toBeTruthy(); + + const [t] = await txns(); + expect(Number(t.amount)).toBe(13.5); + expect(t.source).toBe("wallet"); + expect(t.source_ref).toBe("evt:64"); + // Statementless + card is exactly what puts it in the reconcile queue, with + // no extra wiring: that is how the statement eventually confirms it. + expect(t.statement_id).toBeNull(); + expect(t.payment_method).toBe("card"); + }); + + it("ignores an eBay order-lifecycle notification", async () => { + // 10 of the 13 non-transactions in the capture window are these. + const body = await (await ingest(req(ebayOrderPlaced))).json(); + expect(body.skipped).toBe("not_a_transaction"); + expect(await txns()).toHaveLength(0); + }); + + it("is idempotent on the event id", async () => { + await ingest(req(breadworldBank)); + const second = await (await ingest(req(breadworldBank))).json(); + expect(second.skipped).toBe("already_ingested"); + expect(await txns()).toHaveLength(1); + }); +}); + +describe("the notification pair", () => { + it("merges the two halves into ONE transaction, keeping both contributions", async () => { + await ingest(req(breadworldBank)); // has the card tail + await ingest(req(breadworldWallet)); // has the suburb + + const rows = await txns(); + expect(rows).toHaveLength(1); + // The Wallet half supplies the suburb; the issuer half never had one. + expect(rows[0].location).toBe("Wyndham Vale, VIC"); + + const [m] = await metas(); + // ...and the issuer half's card tail survives the merge. + expect(m.card_last4).toBe("8032"); + expect(m.payment_method_detail).toBe("Mastercard"); + }); + + it("merges in either arrival order", async () => { + await ingest(req(breadworldWallet)); + await ingest(req(breadworldBank)); + + expect(await txns()).toHaveLength(1); + const [m] = await metas(); + expect(m.card_last4).toBe("8032"); + const [t] = await txns(); + expect(t.location).toBe("Wyndham Vale, VIC"); + }); +}); + +describe("currency", () => { + it("records AUD when Amex states it explicitly", async () => { + await ingest(req(amexRewardPay)); + const [t] = await txns(); + expect(t.foreign_currency_code).toBeNull(); + expect(Number(t.amount)).toBe(5103.49); + const [m] = await metas(); + expect(m.currency).toBe("AUD"); + expect(m.flags).not.toContain("currency_unverified"); + }); + + it("flags a bare dollar sign rather than assuming AUD", async () => { + // 14 of 21 captured transaction rows are a bare "$". Assuming AUD is + // harmless at home and a 23x error on a THB tap abroad. + await ingest(req(breadworldBank)); + const [m] = await metas(); + expect(m.currency).toBeNull(); + expect(m.flags).toContain("currency_unverified"); + }); + + it("stores a foreign charge on the NATIVE pair, with a provisional AUD", async () => { + // Seed enough history for the median rate to exist (n >= 5). + const st = await pool.query( + `INSERT INTO statements (filename, account_number, bank_name, billing_start_date, billing_end_date, owner_id) + VALUES ('s.pdf','1','Test Bank','2026-01-01','2026-01-31',$1) RETURNING id`, [ownerId] + ); + for (let i = 0; i < 6; i++) { + await pool.query( + `INSERT INTO transactions (statement_id, transaction_date, description, amount, + transaction_type, owner_id, foreign_currency_amount, foreign_currency_code) + VALUES ($1,'2026-01-10','BANGKOK',4.57,'debit',$2,100,'THB')`, [st.rows[0].id, ownerId] + ); + } + + await ingest(req({ + event_id: 999, kind: "transaction", direction: "debit", amount: 500, + amount_text: "฿500", merchant: "7-Eleven", notif_app: "Wallet", + notif_subtitle: "7-Eleven, Bangkok", occurred_at: "2026-08-20T12:00:00+10:00", + })); + + const [t] = await txns(); + expect(Number(t.foreign_currency_amount)).toBe(500); + expect(t.foreign_currency_code).toBe("THB"); + // The native figure is exact on both sides, so it is what the statement + // will match on. amount is only ever provisional here. + expect(Number(t.amount)).toBeCloseTo(22.85, 1); + const [m] = await metas(); + expect(m.flags).toContain("aud_provisional"); + }); + + it("still records a foreign charge when there is no rate history", async () => { + await ingest(req({ + event_id: 998, kind: "transaction", direction: "debit", amount: 42, + amount_text: "€42.00", merchant: "Somewhere", notif_app: "Wallet", + occurred_at: "2026-08-20T12:00:00+10:00", + })); + const [t] = await txns(); + expect(t.foreign_currency_code).toBe("EUR"); + const [m] = await metas(); + // Says so, rather than inventing a rate. + expect(m.flags).toContain("no_rate_reference"); + }); +}); + +describe("location", () => { + it("takes the suburb from the Wallet subtitle", async () => { + await ingest(req(breadworldWallet)); + const [t] = await txns(); + expect(t.location).toBe("Wyndham Vale, VIC"); + }); + + it("falls back to the fixed-width descriptor, country included", async () => { + // The issuer half has no subtitle; the country code is positional. + await ingest(req({ + ...breadworldBank, event_id: 75, amount: 42.44, + merchant: "SP PEAKDESIGN", + notif_body: "$42.44 paid from Mastercard ending...8032. SP PEAKDESIGN SAN FRANCISOUS", + })); + const [t] = await txns(); + expect(t.location).toBe("SAN FRANCISO US"); + const [m] = await metas(); + // The only foreign merchant in the whole capture window — worth surfacing. + expect(m.flags).toContain("foreign_country_US"); + }); +}); diff --git a/src/__tests__/unit/wallet-descriptor.test.ts b/src/__tests__/unit/wallet-descriptor.test.ts new file mode 100644 index 0000000..2f5e738 --- /dev/null +++ b/src/__tests__/unit/wallet-descriptor.test.ts @@ -0,0 +1,109 @@ +import { describe, it, expect } from "vitest"; +import { + parseWestpacDescriptor, + parseWalletSubtitle, + currencyFromAmountText, +} from "@/lib/wallet-descriptor"; + +/** + * Every input below is a real captured notification body or subtitle from + * capture.wallet_events (2026-08-13 .. 08-22), not an invented fixture. The + * model handled this one fixed-width format three different ways on this exact + * data, which is why the parsing moved into code. + */ + +describe("parseWestpacDescriptor", () => { + it("reads merchant, city and country from the fixed-width block", () => { + expect(parseWestpacDescriptor( + "$13.50 paid from Mastercard ending...8032. Breadworld Bakery Cafe WYNDHAM VAL AU" + )).toEqual({ merchant: "Breadworld Bakery Cafe", city: "WYNDHAM VAL", country: "AU" }); + }); + + it("handles a city padded to the full field width", () => { + expect(parseWestpacDescriptor( + "$54.50 paid from Mastercard ending...8032. PLA DONATION MELBOURNE AU" + )).toEqual({ merchant: "PLA DONATION", city: "MELBOURNE", country: "AU" }); + }); + + it("splits a 12-char city that runs into the country code with no space", () => { + // The case that defeated the model: "SAN FRANCISO" exactly fills the city + // field, so "US" follows immediately. Read as one token it becomes the + // nonsense location "SAN FRANCISOUS" and the country is lost -- on the only + // foreign merchant in the whole capture window. + expect(parseWestpacDescriptor( + "$42.44 paid from Mastercard ending...8032. SP PEAKDESIGN SAN FRANCISOUS" + )).toEqual({ merchant: "SP PEAKDESIGN", city: "SAN FRANCISO", country: "US" }); + }); + + it("reads mixed-case cities the model inconsistently stripped", () => { + expect(parseWestpacDescriptor( + "$20.99 paid from Mastercard ending...8032. Spotify P45AE035CB Sydney AU" + )).toEqual({ merchant: "Spotify P45AE035CB", city: "Sydney", country: "AU" }); + }); + + it("degrades to nulls rather than guessing", () => { + expect(parseWestpacDescriptor(null)).toEqual({ merchant: null, city: null, country: null }); + expect(parseWestpacDescriptor("")).toEqual({ merchant: null, city: null, country: null }); + // No sentence terminator: not this format at all. + expect(parseWestpacDescriptor("You have a new notification")) + .toEqual({ merchant: null, city: null, country: null }); + }); + + it("does not invent a country from a short merchant-only tail", () => { + // "...IT" here is the end of a word, not Italy. A wrong country on a + // foreign transaction is worse than no country. + const r = parseWestpacDescriptor("$9.00 paid from Mastercard ending...8032. SOME MERCHANT"); + expect(r.country).toBeNull(); + }); +}); + +describe("parseWalletSubtitle", () => { + it("separates merchant from suburb and state", () => { + expect(parseWalletSubtitle("Breadworld Bakery Cafe, Wyndham Vale, VIC")) + .toEqual({ merchant: "Breadworld Bakery Cafe", location: "Wyndham Vale, VIC" }); + }); + + it("returns no location for an online merchant", () => { + // 2 of the 10 Wallet subtitles captured have no place at all. + expect(parseWalletSubtitle("Seatsaero")).toEqual({ merchant: "Seatsaero", location: null }); + expect(parseWalletSubtitle("Prism Accounting Bookkeeping Services")) + .toEqual({ merchant: "Prism Accounting Bookkeeping Services", location: null }); + }); + + it("keeps a merchant name containing its own comma out of the location", () => { + expect(parseWalletSubtitle("Big Daddy`s Fresh Ma, Manor Lakes, VIC").location) + .toBe("Manor Lakes, VIC"); + }); + + it("handles absent input", () => { + expect(parseWalletSubtitle(null)).toEqual({ merchant: null, location: null }); + expect(parseWalletSubtitle(" ")).toEqual({ merchant: null, location: null }); + }); +}); + +describe("currencyFromAmountText", () => { + it("reads Amex's explicit marker", () => { + expect(currencyFromAmountText("A$5,103.49")).toBe("AUD"); + expect(currencyFromAmountText("A$13.50")).toBe("AUD"); + }); + + it("refuses to guess a bare dollar sign", () => { + // 14 of 21 captured transaction rows are this. Assuming AUD is harmless at + // home and a 23x error on a THB tap abroad, so the row goes to the + // reconcile queue with no currency instead. + expect(currencyFromAmountText("$13.50")).toBeNull(); + expect(currencyFromAmountText("$103.30")).toBeNull(); + }); + + it("reads unambiguous symbols when a card does state one", () => { + expect(currencyFromAmountText("€24.00")).toBe("EUR"); + expect(currencyFromAmountText("฿500")).toBe("THB"); + expect(currencyFromAmountText("S$18.20")).toBe("SGD"); + expect(currencyFromAmountText("NZ$40.00")).toBe("NZD"); + }); + + it("handles absent input", () => { + expect(currencyFromAmountText(null)).toBeNull(); + expect(currencyFromAmountText("")).toBeNull(); + }); +}); diff --git a/src/app/api/wallet/ingest/route.ts b/src/app/api/wallet/ingest/route.ts new file mode 100644 index 0000000..9f37ffd --- /dev/null +++ b/src/app/api/wallet/ingest/route.ts @@ -0,0 +1,49 @@ +import { NextRequest, NextResponse } from "next/server"; +import { processWalletIngestion, type WalletEvent } from "@/lib/wallet-ingestion"; + +/** + * Machine ingest endpoint for interpreted iOS Wallet notifications. + * + * Sibling to /api/orders/ingest and /api/receipts/ingest, and deliberately + * shaped like them: a shared secret rather than a browser session, because n8n + * calls this app-to-app. + * + * Its own token so the wallet lane's credential can be rotated without touching + * the order flow, which runs on a schedule nobody is watching. + * + * n8n is transport only. The interpretation already happened (Wallet Interpret, + * CAp1s5cuzJk2mDaI, local model on Unraid because card tails are PII); this + * endpoint owns every decision about what reaches the ledger. + */ +function authorised(req: NextRequest): boolean { + const expected = process.env.WALLET_INGEST_TOKEN; + if (!expected) return false; // fail closed when unconfigured + const got = req.headers.get("x-ingest-token"); + return !!got && got === expected; +} + +export async function POST(req: NextRequest) { + if (!authorised(req)) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + + let body: WalletEvent; + try { + body = await req.json(); + } catch { + return NextResponse.json({ error: "invalid JSON" }, { status: 400 }); + } + + if (!Number.isInteger(body?.event_id) || typeof body?.kind !== "string") { + return NextResponse.json({ error: "event_id (int) and kind are required" }, { status: 400 }); + } + + try { + const result = await processWalletIngestion(body); + return NextResponse.json(result); + } catch (e) { + // A 500 here alerts #log-alerts through the workflow error handler, which + // is what should happen: a write that fails is a template or schema change, + // not something to swallow. + const message = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: message }, { status: 500 }); + } +} diff --git a/src/lib/wallet-descriptor.ts b/src/lib/wallet-descriptor.ts new file mode 100644 index 0000000..faa8e7a --- /dev/null +++ b/src/lib/wallet-descriptor.ts @@ -0,0 +1,116 @@ +/** + * Deterministic parsing of the bits of a wallet notification that have a fixed + * shape. + * + * Everything here was previously left to the interpreting model, which handled + * ONE fixed-width format three different ways on the same day's data: it kept + * the country code on "WYNDHAM VAL AU", stripped it on "Sydney" and "Perth", + * and swallowed "SAN FRANCISOUS" whole into the location field. A fixed-width + * field is not a judgement call, and the same argument that put the order + * parsers in code rather than in a Gemini call applies: a regex fails loudly on + * a template change, where a model returns a plausible wrong answer. + * + * The model still decides what the amount and merchant ARE -- it is better at + * that than a pattern, because five cards emit three shapes. This only handles + * the parts that are positional. + */ + +/** + * Westpac writes the merchant/city/country block as a fixed-width statement + * descriptor: 27-char merchant, 12-char city, 2-char ISO country. Verified + * across every Westpac event captured: + * + * Breadworld Bakery Cafe WYNDHAM VAL AU + * PLA DONATION MELBOURNE AU + * SP PEAKDESIGN SAN FRANCISOUS <- "SAN FRANCISO" + "US" + * + * The last example is why this is positional rather than a split on whitespace: + * a 12-character city fills its field exactly and runs straight into the + * country code with no separator. + */ +export type WestpacDescriptor = { + merchant: string | null; + city: string | null; + /** ISO-3166 alpha-2, upper-case. Null when the tail is not a country code. */ + country: string | null; +}; + +const MERCHANT_WIDTH = 27; +const CITY_WIDTH = 12; + +/** + * `body` is the whole notification body, e.g. + * "$13.50 paid from Mastercard ending...8032. Breadworld Bakery Cafe WYNDHAM VAL AU" + * Returns nulls rather than throwing: a template change must degrade to "we + * don't know", never to a wrong country on a foreign transaction. + */ +export function parseWestpacDescriptor(body: string | null | undefined): WestpacDescriptor { + const none: WestpacDescriptor = { merchant: null, city: null, country: null }; + if (!body) return none; + + // The descriptor is whatever follows the first sentence ("... ending...8032. "). + // [\s\S] rather than the `s` flag: the tsconfig target predates es2018. + const m = body.match(/^[\s\S]*?\.\s([\s\S]*)$/); + if (!m) return none; + const tail = m[1].replace(/\s+$/, ""); + if (!tail) return none; + + const merchant = tail.slice(0, MERCHANT_WIDTH).trim() || null; + const rest = tail.slice(MERCHANT_WIDTH); + if (!rest) return { merchant, city: null, country: null }; + + // Country is the final two characters of the fixed block, but only when the + // block is actually long enough to contain one -- a short tail is a merchant + // name with no location, not a merchant whose last two letters are a country. + const block = rest.slice(0, CITY_WIDTH + 2); + if (block.length < CITY_WIDTH + 2) { + return { merchant, city: block.trim() || null, country: null }; + } + const city = block.slice(0, CITY_WIDTH).trim() || null; + const tailTwo = block.slice(CITY_WIDTH).trim().toUpperCase(); + const country = /^[A-Z]{2}$/.test(tailTwo) ? tailTwo : null; + return { merchant, city, country: country }; +} + +/** + * The Apple Wallet notification's subtitle, which is comma-delimited and clean: + * "Breadworld Bakery Cafe, Wyndham Vale, VIC" -> location "Wyndham Vale, VIC" + * "Seatsaero" -> no location (online merchant) + * + * Only the first comma separates merchant from place; the rest is the place. + */ +export function parseWalletSubtitle(subtitle: string | null | undefined): { + merchant: string | null; + location: string | null; +} { + if (!subtitle || !subtitle.trim()) return { merchant: null, location: null }; + const parts = subtitle.split(",").map((p) => p.trim()).filter(Boolean); + if (parts.length <= 1) return { merchant: parts[0] ?? null, location: null }; + return { merchant: parts[0], location: parts.slice(1).join(", ") }; +} + +/** + * Currency marker, if the notification states one. + * + * Only Amex does: it writes "A$5,103.49". Westpac, CommBank and Apple Wallet + * write a bare "$", which is honestly recorded as unknown rather than assumed + * to be AUD -- 14 of 21 captured transaction rows. Domestically that assumption + * would be harmless; abroad it is a 23x error on a THB tap, so it is not made + * here. The statement settles it within weeks, and the reconcile queue holds + * the row until then. + */ +export function currencyFromAmountText(amountText: string | null | undefined): string | null { + if (!amountText) return null; + const t = amountText.trim(); + if (/^A\$/i.test(t)) return "AUD"; + if (/^(NZ\$)/i.test(t)) return "NZD"; + if (/^US\$/i.test(t)) return "USD"; + if (/^€/.test(t)) return "EUR"; + if (/^£/.test(t)) return "GBP"; + if (/^¥/.test(t)) return "JPY"; + if (/^฿/.test(t)) return "THB"; + if (/^S\$/i.test(t)) return "SGD"; + if (/^₹/.test(t)) return "INR"; + // A bare "$" is genuinely ambiguous -- do not guess. + return null; +} diff --git a/src/lib/wallet-ingestion.ts b/src/lib/wallet-ingestion.ts new file mode 100644 index 0000000..5d29f6e --- /dev/null +++ b/src/lib/wallet-ingestion.ts @@ -0,0 +1,228 @@ +import { queryRow, queryRaw } from "@/lib/db"; +import { DEFAULT_OWNER_ID } from "@/lib/order-ingestion"; +import { + parseWestpacDescriptor, + parseWalletSubtitle, + currencyFromAmountText, +} from "@/lib/wallet-descriptor"; + +/** + * Wallet notifications -> transactions. + * + * A transaction is created for EVERY tap (DECISIONS.md ING-11, 2026-08-22), + * because overseas trips are where live spend matters most: the statement lag + * is the same 6-35 days abroad but knowing what you have spent matters far more + * in an unfamiliar currency. + * + * SEQUENCING IS THE WHOLE DESIGN. Wallet is the FIRST signal, not a late one -- + * measured on txn 6858, the notification arrived 35h25m before anything created + * a transaction. So this creates the row, the receipt enriches it + * (receipt-ingestion.ts adopts it), and the statement confirms it via the + * reconcile queue, which a wallet row enters unaided: it is statementless with + * payment_method 'card'. Nothing here needs to know about statements. + */ + +export type WalletEvent = { + /** capture.wallet_events.id -- the idempotency key. */ + event_id: number; + /** 'transaction' | 'not_transaction' | 'unsure' from the interpreter. */ + kind: string; + direction?: string | null; + amount: number | null; + /** The exact substring the model read the amount from, e.g. "A$5,103.49". */ + amount_text?: string | null; + currency?: string | null; + merchant?: string | null; + card_tail?: string | null; + issuer?: string | null; + location?: string | null; + notif_app?: string | null; + notif_body?: string | null; + notif_subtitle?: string | null; + occurred_at?: string | null; +}; + +export type WalletIngestResult = { + transactionId: number | null; + metadataId: number | null; + skipped?: "not_a_transaction" | "already_ingested" | "no_amount" | "merged_pair"; + flags: string[]; +}; + +/** Days either side within which a second notification is the SAME tap. */ +const PAIR_WINDOW_DAYS = 1; + +/** + * A ledger-derived AUD rate for a currency: the median of what the bank + * actually charged divided by the native figure, over statement rows. + * + * Same source order_transaction_bridge uses for its plausibility guard. This is + * only ever a PROVISIONAL value -- transactions.amount is NOT NULL and always + * AUD, and a notification cannot know the AUD figure because the bank sets the + * rate at settlement. The statement overwrites it at reconcile time. + * Returns null when there is not enough history to be worth anything. + */ +async function medianRateToAud(code: string): Promise { + const row = await queryRow<{ rate: string | null }>( + `SELECT percentile_cont(0.5) WITHIN GROUP ( + ORDER BY amount / NULLIF(foreign_currency_amount, 0) + )::text AS rate + FROM transactions + WHERE foreign_currency_code = $1 + AND foreign_currency_amount > 0 + AND superseded_by_id IS NULL + HAVING count(*) >= 5`, + [code] + ); + const rate = row?.rate ? Number(row.rate) : null; + return rate && Number.isFinite(rate) && rate > 0 ? rate : null; +} + +/** + * The other half of a paired notification, if it has already been ingested. + * + * One tap emits TWO notifications ~2-8s apart: the issuer app (which has the + * card tail) and Apple Wallet (which has the better merchant and the suburb). + * Neither alone is complete, so they must be MERGED, never deduped -- dropping + * either loses a field nothing else supplies. Matched on amount and date rather + * than any id, because the notifications share no identifier. + */ +async function findPairedRow(amount: number, date: string) { + const rows = await queryRaw<{ id: number; meta_id: number | null }>( + `SELECT t.id, em.id AS meta_id + FROM transactions t + LEFT JOIN expense_metadata em ON em.transaction_id = t.id + WHERE t.source = 'wallet' + AND t.statement_id IS NULL + AND t.amount = $1::numeric + AND t.transaction_date BETWEEN $2::date - $3::int AND $2::date + $3::int + AND t.superseded_by_id IS NULL + LIMIT 2`, + [amount, date, PAIR_WINDOW_DAYS] + ); + return rows.length === 1 ? rows[0] : null; +} + +export async function processWalletIngestion(ev: WalletEvent): Promise { + const flags: string[] = []; + + // An open-world classifier feeding a closed-world writer drops the unknown + // and calls it a refusal (ING-14). `unsure` is a real answer and must not + // silently become a transaction. + if (ev.kind !== "transaction") { + return { transactionId: null, metadataId: null, skipped: "not_a_transaction", flags }; + } + if (ev.amount === null || !Number.isFinite(ev.amount) || ev.amount <= 0) { + return { transactionId: null, metadataId: null, skipped: "no_amount", flags }; + } + + const sourceRef = `evt:${ev.event_id}`; + const existing = await queryRow<{ id: number }>( + `SELECT id FROM transactions WHERE source = 'wallet' AND source_ref = $1`, + [sourceRef] + ); + if (existing) { + return { transactionId: existing.id, metadataId: null, skipped: "already_ingested", flags }; + } + + const date = (ev.occurred_at ?? new Date().toISOString()).slice(0, 10); + + // Deterministic parsing of the positional fields. The model decides what the + // amount and merchant ARE; it does not get to parse a fixed-width block. + const desc = parseWestpacDescriptor(ev.notif_body); + const sub = parseWalletSubtitle(ev.notif_subtitle); + const merchant = sub.merchant ?? ev.merchant ?? desc.merchant ?? "Card purchase"; + // Two sources for a place, and they are not equal. The Apple Wallet subtitle + // gives "Wyndham Vale, VIC"; the issuer's fixed-width descriptor gives + // "WYNDHAM VAL AU" -- a 12-character truncation in caps. The subtitle wins + // outright, which matters for the pair merge below: whichever half arrives + // second must be able to REPLACE a descriptor-derived place, or the result + // depends on arrival order. + const subtitleLocation = sub.location ?? null; + const location = subtitleLocation + ?? ev.location + ?? (desc.city && desc.country ? `${desc.city} ${desc.country}` : desc.city); + + // Currency: only ever what the notification actually states. A bare "$" is + // recorded as unknown, never assumed AUD -- harmless domestically, a 23x + // error on a THB tap abroad. + const stated = currencyFromAmountText(ev.amount_text) + ?? (ev.currency && /^[A-Z]{3}$/.test(ev.currency) ? ev.currency : null); + if (!stated) flags.push("currency_unverified"); + if (desc.country && desc.country !== "AU") flags.push(`foreign_country_${desc.country}`); + + // Foreign: the native figure is what both sides can agree on exactly, so it + // is what gets stored. `amount` still needs a value (NOT NULL) and takes a + // provisional conversion, flagged, which the statement overwrites. + let amountAud = ev.amount; + let fxAmount: number | null = null; + let fxCode: string | null = null; + if (stated && stated !== "AUD") { + fxAmount = ev.amount; + fxCode = stated; + const rate = await medianRateToAud(stated); + if (rate) { + amountAud = Number((ev.amount * rate).toFixed(2)); + flags.push("aud_provisional"); + } else { + flags.push("aud_provisional", "no_rate_reference"); + } + } + + // Merge the paired notification rather than creating a second row for one + // tap. The issuer half brings the card tail; the Wallet half brings the + // merchant and suburb. COALESCE so whichever arrives second only ever fills + // gaps -- it can never blank a field the first half supplied. + const pair = await findPairedRow(amountAud, date); + if (pair) { + await queryRaw( + `UPDATE transactions + SET merchant_name = COALESCE(NULLIF($2,'Card purchase'), merchant_name), + merchant_normalized = COALESCE(NULLIF($2,'Card purchase'), merchant_normalized), + -- A subtitle-derived place overwrites; anything else only fills a + -- gap. Without the $4 branch the merged row depends on which + -- notification happened to arrive first, and the issuer half + -- usually wins by a few seconds -- keeping the worse string. + location = CASE WHEN $4::boolean THEN COALESCE($3, location) + ELSE COALESCE(location, $3) END + WHERE id = $1`, + [pair.id, merchant, location, subtitleLocation !== null] + ); + if (pair.meta_id) { + await queryRaw( + `UPDATE expense_metadata + SET card_last4 = COALESCE(card_last4, $2), + payment_method_detail = COALESCE(payment_method_detail, $3), + merchant_normalized = COALESCE(NULLIF($4,'Card purchase'), merchant_normalized) + WHERE id = $1`, + [pair.meta_id, ev.card_tail ?? null, ev.issuer ?? null, merchant] + ); + } + return { transactionId: pair.id, metadataId: pair.meta_id, skipped: "merged_pair", flags }; + } + + const txn = await queryRow<{ id: number }>( + `INSERT INTO transactions ( + transaction_date, description, amount, amount_aud, category, payment_method, + merchant_name, merchant_normalized, transaction_type, owner_id, location, + foreign_currency_amount, foreign_currency_code, source, source_ref, row_index + ) VALUES ($1,$2,$3,$3,'other','card',$4,$4,'debit',$5,$6,$7,$8,'wallet',$9,0) + RETURNING id`, + [date, merchant, amountAud, merchant, DEFAULT_OWNER_ID, location, fxAmount, fxCode, sourceRef] + ); + + const meta = await queryRow<{ id: number }>( + `INSERT INTO expense_metadata ( + transaction_id, source, source_message_id, merchant_normalized, amount, + transaction_date, card_last4, currency, payment_method, + payment_method_detail, line_items, flags, extraction_model + ) VALUES ($1,'wallet',$2,$3,$4,$5,$6,$7,'card',$8,'[]'::jsonb,$9::jsonb,'wallet-interpret') + RETURNING id`, + [ + txn!.id, sourceRef, merchant, ev.amount, date, + ev.card_tail ?? null, stated, ev.issuer ?? null, JSON.stringify(flags), + ] + ); + + return { transactionId: txn!.id, metadataId: meta!.id, flags }; +}