diff --git a/scripts/import-frollo.mts b/scripts/import-frollo.mts new file mode 100644 index 0000000..cc95ce5 --- /dev/null +++ b/scripts/import-frollo.mts @@ -0,0 +1,259 @@ +/** + * Imports a Frollo transaction export into `transactions`. + * + * node --experimental-strip-types scripts/import-frollo.mts --file + * node --experimental-strip-types scripts/import-frollo.mts --file --apply + * + * Dry run by default: it prints exactly what it would do and touches nothing. + * Rehearse first — every irreversible step in this repo that skipped that has + * cost something, and the failure mode here is doubling reported income. + * + * Export the file with PENDING TRANSACTIONS EXCLUDED. A pending row changes both + * its id and its description when it settles, so importing one guarantees a + * duplicate on the next run; and every pending row observed so far has been on a + * credit card, which this importer does not cover anyway. See DECISIONS.md + * ING-11 in the smarthome repo. + * + * Needs DATABASE_URL. postgres-personal publishes no host port, so from the host: + * export DATABASE_URL="postgresql://personal:@$(docker inspect postgres-personal \ + * --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):5432/personal" + * The container IP changes on every recreate. + */ +import { readFileSync } from "node:fs"; +import pg from "pg"; +import { parseCSVRows } from "../src/lib/csv-parser.ts"; +import { + accountFor, + accountReport, + dedupe, + readRows, + skipReason, + toLedgerRow, + unknownAccounts, + type FrolloRow, + type SkipReason, +} from "../src/lib/frollo-csv.ts"; + +const SOURCE = "frollo"; + +function arg(name: string, fallback?: string): string | undefined { + const i = process.argv.indexOf(`--${name}`); + if (i >= 0 && process.argv[i + 1] && !process.argv[i + 1].startsWith("--")) { + return process.argv[i + 1]; + } + return fallback; +} +const has = (name: string) => process.argv.includes(`--${name}`); + +const file = arg("file"); +const apply = has("apply"); +const ownerId = Number(arg("owner", "1")); +const allowMissing = has("allow-missing-accounts"); + +if (!file) { + console.error("usage: --file [--apply] [--owner ] [--allow-missing-accounts]"); + process.exit(2); +} + +const money = (n: number) => n.toLocaleString("en-AU", { minimumFractionDigits: 2, maximumFractionDigits: 2 }); + +const rows = readRows(parseCSVRows(readFileSync(file, "utf8"))); +console.log(`\n${file}`); +console.log(` ${rows.length} rows in file\n`); + +// ---- 1. scope ------------------------------------------------------------ +const skipped = new Map(); +const inScope: FrolloRow[] = []; +for (const r of rows) { + const why = skipReason(r); + if (why) { + const list = skipped.get(why); + if (list) list.push(r); + else skipped.set(why, [r]); + } else inScope.push(r); +} +console.log("SCOPE"); +for (const [why, list] of [...skipped.entries()].sort((a, b) => b[1].length - a[1].length)) { + console.log(` skipped ${String(list.length).padStart(5)} ${why}`); +} +console.log(` in scope ${String(inScope.length).padStart(4)}\n`); + +if (skipped.has("pending")) { + console.log( + ` NOTE: ${skipped.get("pending")!.length} pending row(s) present — the export was taken\n` + + ` with pending INCLUDED. They are skipped, but re-export with pending\n` + + ` excluded so successive files are comparable.\n` + ); +} + +// ---- 2. accounts --------------------------------------------------------- +// A lapsed CDR consent removes an account from the export silently; the row +// count just drops. Assert the roster rather than trusting it. +console.log("ACCOUNTS"); +const report = accountReport(inScope); +for (const a of report) { + const mark = a.present ? " " : "!"; + console.log(` ${mark} ${a.spec.last4} ${a.spec.label.padEnd(24)} ${String(a.rows).padStart(5)} rows`); +} +const missing = report.filter((a) => !a.present); +const unknown = unknownAccounts(rows); +if (unknown.length > 0) { + console.log("\n accounts in the file this importer does not know (skipped):"); + for (const u of unknown) console.log(` ${u.accountNumber} ${u.accountName} ${u.rows} rows`); +} +if (missing.length > 0) { + console.log(`\n ${missing.length} configured account(s) contributed NO rows:`); + for (const m of missing) console.log(` ${m.spec.last4} ${m.spec.label} (${m.spec.provider})`); + console.log(" A CDR consent may have lapsed — check before treating this file as complete."); + if (!allowMissing) { + console.log(" Refusing to continue. Re-run with --allow-missing-accounts if this is expected.\n"); + process.exit(1); + } +} +console.log(); + +// ---- 3. dedupe ----------------------------------------------------------- +const { kept, dropped } = dedupe(inScope); +console.log("DE-DUPLICATION (re-ingest twins from CDR re-consent)"); +console.log(` ${dropped.length} dropped, ${kept.length} kept`); +if (dropped.length > 0) { + const byAccount = new Map(); + for (const d of dropped) { + const label = accountFor(d.row.accountNumber)?.label ?? "?"; + byAccount.set(label, (byAccount.get(label) ?? 0) + 1); + } + for (const [label, n] of [...byAccount.entries()].sort((a, b) => b[1] - a[1])) { + console.log(` ${String(n).padStart(4)} ${label}`); + } + console.log(" examples:"); + for (const d of dropped.slice(0, 3)) { + console.log( + ` ${d.row.transactionDate} ${money(d.row.amount).padStart(11)} ${d.row.description.slice(0, 34)}` + + ` id ${d.row.transactionId} duplicates ${d.duplicateOf.transactionId} (gap ${d.idGap})` + ); + } + // Ids this close look like a genuine same-day repeat rather than a twin. + // Nothing in scope has ever tripped it; if something does, look before + // trusting the collapse. + const suspect = dropped.filter((d) => d.suspectGenuineRepeat); + if (suspect.length > 0) { + console.log(`\n REVIEW: ${suspect.length} collapsed row(s) had near-consecutive ids and may be`); + console.log(" real repeats rather than duplicates:"); + for (const d of suspect.slice(0, 10)) { + console.log( + ` ${d.row.transactionDate} ${money(d.row.amount).padStart(11)} ${d.row.description.slice(0, 40)}` + + ` ids ${d.duplicateOf.transactionId}/${d.row.transactionId} gap ${d.idGap}` + ); + } + } +} +console.log(); + +// ---- 4. what would change ------------------------------------------------ +const ledger = kept.map(toLedgerRow); +if (!process.env.DATABASE_URL) { + console.error("DATABASE_URL is not set — cannot compare against the ledger."); + process.exit(2); +} +const client = new pg.Client({ connectionString: process.env.DATABASE_URL }); +await client.connect(); + +const existing = await client.query<{ source_ref: string }>( + "SELECT source_ref FROM transactions WHERE source = $1", + [SOURCE] +); +const seen = new Set(existing.rows.map((r) => r.source_ref)); +const fresh = ledger.filter((r) => !seen.has(r.sourceRef)); +const already = ledger.length - fresh.length; + +console.log("LEDGER"); +console.log(` already imported : ${String(already).padStart(5)}`); +console.log(` new to insert : ${String(fresh.length).padStart(5)}`); +const debits = fresh.filter((r) => r.transactionType === "debit"); +const credits = fresh.filter((r) => r.transactionType === "credit"); +const aud = (rs: typeof fresh) => rs.filter((r) => !r.foreignCurrencyCode); +console.log( + ` of which AUD : ${aud(debits).length} debits ${money(aud(debits).reduce((s, r) => s + r.amount, 0))}` + + ` / ${aud(credits).length} credits ${money(aud(credits).reduce((s, r) => s + r.amount, 0))}` +); +const fx = fresh.filter((r) => r.foreignCurrencyCode); +if (fx.length > 0) { + const byCcy = new Map(); + for (const r of fx) byCcy.set(r.foreignCurrencyCode!, (byCcy.get(r.foreignCurrencyCode!) ?? 0) + 1); + console.log(` foreign : ${[...byCcy].map(([c, n]) => `${n} ${c}`).join(", ")} (amount_aud left NULL)`); +} +if (fresh.length > 0) { + console.log("\n first rows to insert:"); + for (const r of fresh.slice(0, 8)) { + console.log( + ` ${r.transactionDate} ${r.transactionType.padEnd(6)} ${money(r.amount).padStart(11)}` + + ` ${(r.foreignCurrencyCode ?? "AUD").padEnd(4)} ${r.description.slice(0, 40)}` + ); + } +} + +// ---- 5. apply ------------------------------------------------------------ +if (!apply) { + console.log("\nDRY RUN — nothing written. Re-run with --apply to insert.\n"); + await client.end(); + process.exit(0); +} + +if (fresh.length === 0) { + console.log("\nNothing to insert.\n"); + await client.end(); + process.exit(0); +} + +// row_index is left NULL: it exists to order statement lines, and the +// uniqueness this import relies on is uq_transaction_source_ref, not +// uq_transaction_identity. +// +// category is not set, because Frollo's own categories are not trustworthy. Note +// what actually lands: trg_transactions_normalize_category rewrites NULL to +// 'other' on insert, so these rows arrive categorised 'other' rather than +// uncategorised. That is equivalent for display and for the rules engine, which +// writes transaction_overrides.category_override and is read ahead of +// t.category by EFFECTIVE_CATEGORY — so a rule still wins. +let inserted = 0; +try { + await client.query("BEGIN"); + for (const r of fresh) { + const res = await client.query( + `INSERT INTO transactions + (statement_id, owner_id, transaction_date, description, amount, transaction_type, + merchant_name, foreign_currency_amount, foreign_currency_code, + source, source_ref, source_account) + VALUES (NULL, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT (source, source_ref) WHERE source IS NOT NULL AND source_ref IS NOT NULL + DO NOTHING`, + [ + ownerId, + r.transactionDate, + r.description, + r.amount, + r.transactionType, + r.merchantName, + r.foreignCurrencyAmount, + r.foreignCurrencyCode, + SOURCE, + r.sourceRef, + r.sourceAccount, + ] + ); + inserted += res.rowCount ?? 0; + } + await client.query("COMMIT"); +} catch (e) { + await client.query("ROLLBACK"); + console.error("\nROLLED BACK:", e); + await client.end(); + process.exit(1); +} + +const after = await client.query<{ n: string }>( + "SELECT count(*) AS n FROM transactions WHERE source = $1", + [SOURCE] +); +console.log(`\nInserted ${inserted} row(s). ${after.rows[0].n} Frollo rows now in the ledger.\n`); +await client.end(); diff --git a/src/__tests__/unit/frollo-csv.test.ts b/src/__tests__/unit/frollo-csv.test.ts new file mode 100644 index 0000000..bbd2aac --- /dev/null +++ b/src/__tests__/unit/frollo-csv.test.ts @@ -0,0 +1,309 @@ +import { describe, it, expect } from "vitest"; +import { parseCSVRows } from "@/lib/csv-parser"; +import { + ACCOUNTS, + SUSPECT_REPEAT_GAP, + accountFor, + accountReport, + dedupe, + last4, + naturalKey, + readRows, + skipReason, + toLedgerRow, + unknownAccounts, + type FrolloRow, +} from "@/lib/frollo-csv"; + +const HEADER = + "transaction_id,description,user_description,amount,currency,transaction_date,posted_date," + + "account_number,account_name,credit_debit,transaction_type,provider_name,merchant_name," + + "budget_category,category_name,user_tags,notes,included"; + +function csv(...lines: string[]): FrolloRow[] { + return readRows(parseCSVRows([HEADER, ...lines].join("\n"))); +} + +/** Shapes a line in the export's real column order. */ +function line(o: { + id: string; + desc?: string; + amount: string; + currency?: string; + date?: string; + posted?: string; + account?: string; + accountName?: string; + cd?: string; + type?: string; + merchant?: string; +}): string { + return [ + o.id, + o.desc ?? "SOME MERCHANT", + "", + o.amount, + o.currency ?? "AUD", + o.date ?? "2026-03-01", + o.posted ?? "", + o.account ?? "xxx-xxx xxxx1910", + o.accountName ?? "Smart Access", + o.cd ?? (o.amount.startsWith("-") ? "debit" : "credit"), + o.type ?? "payment", + "CommBank", + o.merchant ?? "Some Merchant", + "lifestyle", + "Groceries", + '""', + "", + "true", + ].join(","); +} + +describe("readRows", () => { + it("maps by header name, not position", () => { + const rows = csv(line({ id: "1", amount: "-12.34", desc: "COFFEE" })); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + transactionId: "1", + description: "COFFEE", + amount: -12.34, + currency: "AUD", + accountName: "Smart Access", + }); + }); + + it("throws when a required column is missing rather than reading undefined", () => { + const broken = "transaction_id,description,amount\n1,X,-1.00"; + expect(() => readRows(parseCSVRows(broken))).toThrow(/missing expected column/i); + }); + + it("keeps a quoted description containing commas intact", () => { + const desc = '"DIDI_NZ DidiNZ pending Auckland NZL, ##0626 8.77 NZ DOLLAR"'; + const rows = csv(line({ id: "1", amount: "-7.23", desc })); + expect(rows[0].description).toContain("NZL, ##0626"); + }); + + it("treats an absent posted_date as null, not empty string", () => { + expect(csv(line({ id: "1", amount: "-1.00" }))[0].postedDate).toBeNull(); + }); +}); + +describe("account scoping", () => { + it("matches on the last four digits regardless of masking format", () => { + expect(last4("xxx-xxx xxxx1910")).toBe("1910"); + expect(last4("xxxxxx xxxxx6264")).toBe("6264"); + expect(last4("x xxxx 0887 ")).toBe("0887"); + expect(last4("XXXXXXXXXXXX0351")).toBe("0351"); + }); + + it("has no duplicate last4 among configured accounts", () => { + const keys = ACCOUNTS.map((a) => a.last4); + expect(new Set(keys).size).toBe(keys.length); + }); + + it("excludes the credit cards, which statements already cover", () => { + for (const card of ["3893", "8032", "1000", "0351", "6227"]) { + expect(ACCOUNTS.find((a) => a.last4 === card)).toBeUndefined(); + } + }); + + it("recognises the loan and income accounts", () => { + expect(accountFor("xxx-xxx xxxxx0682")?.label).toMatch(/loan/i); + expect(accountFor("xxxxxxxxxxxx4830")?.label).toMatch(/income/i); + }); +}); + +describe("skipReason", () => { + it("skips a zero-amount fee waiver", () => { + const row = csv( + line({ id: "1", amount: "0.0", desc: "INTNL TRANSACTION FEE, $ 41.58 FEE SAV" }) + )[0]; + expect(skipReason(row)).toBe("zero_amount"); + }); + + it("skips a credit-card row", () => { + const row = csv(line({ id: "1", amount: "-50.00", account: "xxxx xxxx xxxx 3893" }))[0]; + expect(skipReason(row)).toBe("out_of_scope_account"); + }); + + it("skips a Westpac pending row on its description prefix", () => { + const row = csv( + line({ id: "1", amount: "-13.50", desc: "Pending - Breadworld Bakery Cafe", account: "xxx-xxx xx4878" }) + )[0]; + expect(skipReason(row)).toBe("pending"); + }); + + it("accepts an ordinary in-scope row", () => { + expect(skipReason(csv(line({ id: "1", amount: "-29.17" }))[0])).toBeNull(); + }); + + it("skips rather than imports an unparseable amount", () => { + const row = csv(line({ id: "1", amount: "n/a" }))[0]; + expect(skipReason(row)).toBe("unparseable_amount"); + }); +}); + +describe("dedupe", () => { + it("collapses a re-ingest twin to the earlier id", () => { + const rows = csv( + line({ id: "1200000000", amount: "-61.03", desc: "AMAZON MARKETPLACE" }), + line({ id: "1326000000", amount: "-61.03", desc: "AMAZON MARKETPLACE" }) + ); + const { kept, dropped } = dedupe(rows); + expect(kept).toHaveLength(1); + expect(kept[0].transactionId).toBe("1200000000"); + expect(dropped).toHaveLength(1); + expect(dropped[0].duplicateOf.transactionId).toBe("1200000000"); + }); + + it("collapses close-id repeats too, but flags them for review", () => { + // Five identical $1.00 Apple charges with consecutive ids. On a card + // account, so out of scope in practice — but if such a group ever reaches + // dedupe it must be surfaced, not silently decided either way. + const rows = csv( + ...["1326128755", "1326128756", "1326128757", "1326128758", "1326128759"].map((id) => + line({ id, amount: "-1.0", desc: "APPLE.COM/BILL SYDNEY", date: "2026-07-28" }) + ) + ); + const { kept, dropped } = dedupe(rows); + expect(kept).toHaveLength(1); + expect(dropped).toHaveLength(4); + expect(dropped.every((d) => d.suspectGenuineRepeat)).toBe(true); + }); + + it("does not flag a far-apart twin as a possible repeat", () => { + const rows = csv( + line({ id: "1101410977", amount: "10790.8", currency: "USD", account: "xxxxxxxxxxxx4830" }), + line({ id: "1101415475", amount: "10790.8", currency: "USD", account: "xxxxxxxxxxxx4830" }) + ); + const { kept, dropped } = dedupe(rows); + expect(kept).toHaveLength(1); + expect(dropped[0].suspectGenuineRepeat).toBe(false); + expect(dropped[0].idGap).toBe(4498); + }); + + it("collapses a real income twin — the 100%-overstatement case", () => { + // Every HDR salary payment appears twice in every export, id gaps ranging + // from 58 to 79 million. Keeping both doubled reported income. + const rows = csv( + line({ id: "1126624097", amount: "10790.8", currency: "USD", account: "xxxxxxxxxxxx4830", desc: "HDR Global Services" }), + line({ id: "1126624242", amount: "10790.8", currency: "USD", account: "xxxxxxxxxxxx4830", desc: "HDR Global Services" }) + ); + const { kept } = dedupe(rows); + expect(kept).toHaveLength(1); + expect(kept[0].transactionId).toBe("1126624097"); + }); + + it("does not merge rows that differ in amount, date, description or account", () => { + const rows = csv( + line({ id: "1000000001", amount: "-10.00" }), + line({ id: "1326000001", amount: "-10.01" }), + line({ id: "1326000002", amount: "-10.00", date: "2026-03-02" }), + line({ id: "1326000003", amount: "-10.00", desc: "OTHER" }), + line({ id: "1326000004", amount: "-10.00", account: "xxx-xxx xx4878" }) + ); + expect(dedupe(rows).kept).toHaveLength(5); + }); + + it("keeps the lowest id, because old ids survive a re-ingest", () => { + const rows = csv( + line({ id: "1326000001", amount: "-5.00" }), + line({ id: "1000000000", amount: "-5.00" }), + line({ id: "1200000000", amount: "-5.00" }) + ); + const { kept } = dedupe(rows); + expect(kept).toHaveLength(1); + expect(kept[0].transactionId).toBe("1000000000"); + }); + + it("reports the id gap so a suspect group can be reviewed", () => { + const near = csv( + line({ id: "1000000000", amount: "-5.00" }), + line({ id: String(1000000000 + SUSPECT_REPEAT_GAP - 1), amount: "-5.00" }) + ); + expect(dedupe(near).dropped[0].suspectGenuineRepeat).toBe(true); + const far = csv( + line({ id: "1000000000", amount: "-5.00" }), + line({ id: String(1000000000 + SUSPECT_REPEAT_GAP), amount: "-5.00" }) + ); + expect(dedupe(far).dropped[0].suspectGenuineRepeat).toBe(false); + }); + + it("is idempotent", () => { + const rows = csv( + line({ id: "1000000001", amount: "-61.03" }), + line({ id: "1326000001", amount: "-61.03" }) + ); + const once = dedupe(rows).kept; + expect(dedupe(once).kept).toEqual(once); + }); +}); + +describe("naturalKey", () => { + it("is stable across masking differences on the same account", () => { + const a = csv(line({ id: "1", amount: "-1.00", account: "xxx-xxx xxxx1910" }))[0]; + const b = csv(line({ id: "2", amount: "-1.00", account: "xx-xxx xxxxxx1910" }))[0]; + expect(naturalKey(a)).toBe(naturalKey(b)); + }); +}); + +describe("toLedgerRow", () => { + it("stores a debit as a positive amount with a direction", () => { + const r = toLedgerRow(csv(line({ id: "1", amount: "-29.17" }))[0]); + expect(r.amount).toBe(29.17); + expect(r.transactionType).toBe("debit"); + }); + + it("marks a positive amount as a credit", () => { + const r = toLedgerRow(csv(line({ id: "1", amount: "302.70" }))[0]); + expect(r.amount).toBe(302.7); + expect(r.transactionType).toBe("credit"); + }); + + it("leaves a foreign row's amount native and names the currency", () => { + const r = toLedgerRow( + csv(line({ id: "1", amount: "-10001.13", currency: "USD", account: "xxxxxxxxxxxx4830" }))[0] + ); + expect(r.amount).toBe(10001.13); + expect(r.foreignCurrencyCode).toBe("USD"); + expect(r.foreignCurrencyAmount).toBe(10001.13); + }); + + it("carries no foreign fields for an AUD row", () => { + const r = toLedgerRow(csv(line({ id: "1", amount: "-5.00" }))[0]); + expect(r.foreignCurrencyCode).toBeNull(); + expect(r.foreignCurrencyAmount).toBeNull(); + }); + + it("drops Frollo's literal 'Unknown' merchant rather than showing it", () => { + const r = toLedgerRow(csv(line({ id: "1", amount: "-5.00", merchant: "Unknown" }))[0]); + expect(r.merchantName).toBeNull(); + }); + + it("keeps a real merchant", () => { + const r = toLedgerRow(csv(line({ id: "1", amount: "-5.00", merchant: "Solar Victoria" }))[0]); + expect(r.merchantName).toBe("Solar Victoria"); + }); + + it("carries the provider's id as source_ref for idempotency", () => { + const r = toLedgerRow(csv(line({ id: "986846710", amount: "-5.00" }))[0]); + expect(r.sourceRef).toBe("986846710"); + }); +}); + +describe("accountReport / unknownAccounts", () => { + it("reports a configured account with no rows as absent — a lapsed consent", () => { + const rows = csv(line({ id: "1", amount: "-5.00", account: "xxx-xxx xxxx1910" })); + const report = accountReport(rows); + expect(report.find((r) => r.spec.last4 === "1910")?.present).toBe(true); + expect(report.find((r) => r.spec.last4 === "4830")?.present).toBe(false); + }); + + it("surfaces an account in the file that the importer does not know", () => { + const rows = csv(line({ id: "1", amount: "-5.00", account: "xxxx xxxx xxxx 9999", accountName: "New Card" })); + const unknown = unknownAccounts(rows); + expect(unknown).toHaveLength(1); + expect(unknown[0]).toMatchObject({ accountName: "New Card", rows: 1 }); + }); +}); diff --git a/src/lib/frollo-csv.ts b/src/lib/frollo-csv.ts new file mode 100644 index 0000000..24294dd --- /dev/null +++ b/src/lib/frollo-csv.ts @@ -0,0 +1,347 @@ +/** + * Frollo transaction-export parsing and de-duplication. + * + * Frollo is a CDR (Open Banking) aggregator. Its app emails a CSV of up to 12 + * months of transactions. This module turns that file into rows ready for + * `transactions`, and — mostly — decides which of them are real. + * + * Scope is deliberately narrow: credit cards are excluded because they already + * arrive as monthly statements, which are authoritative. What Frollo is for is + * the accounts whose statements arrive every 182 to 460 days (see + * ACCOUNTS below), where the alternative is downloading each one by hand. + * + * The file has three properties that any consumer has to handle, all learned by + * diffing three real exports (smarthome DECISIONS.md ING-11): + * + * 1. A CDR re-consent makes Frollo re-ingest the account's whole history under + * fresh transaction ids, while the originals survive. So the same purchase + * appears twice with different ids, and consents expire annually — this is + * not a one-off. On one 1,989-row export, 385 rows were such twins, and Wise + * credits were overstated by exactly 100%. + * 2. `transaction_id` is otherwise stable: across two exports 12 days apart, + * 1,462 of 1,532 shared rows kept identical ids and none were replaced. It + * is therefore a good `source_ref` — it just cannot detect a twin. + * 3. The export cannot express "pending", and the tell differs per bank + * (Westpac prefixes the description; Amex emits no `posted_date` or + * `transaction_type` at all, so its pending rows are indistinguishable from + * its posted ones). The export's own option is the only reliable control. + * Every pending row observed was on a credit card, i.e. out of scope, so + * exporting with pending EXCLUDED costs nothing here — and it avoids the + * re-key problem, since a settling row changes both its id and its + * description. + */ + +/** A row as it appears in the CSV, after header mapping. */ +export interface FrolloRow { + transactionId: string; + description: string; + amount: number; + currency: string; + transactionDate: string; + postedDate: string | null; + accountNumber: string; + accountName: string; + creditDebit: string; + transactionType: string | null; + providerName: string; + merchantName: string | null; +} + +/** A row shaped for insertion into `transactions`. */ +export interface LedgerRow { + sourceRef: string; + sourceAccount: string; + transactionDate: string; + description: string; + /** Always positive; direction lives in `transactionType`. */ + amount: number; + transactionType: "debit" | "credit"; + merchantName: string | null; + foreignCurrencyAmount: number | null; + foreignCurrencyCode: string | null; +} + +export interface AccountSpec { + /** Last four characters of the digits in the masked account number. */ + last4: string; + label: string; + provider: string; +} + +/** + * Accounts this importer owns. + * + * Keyed on the last four digits rather than the masked string, because the + * masking format varies by institution (`xxx-xxx xxxx1910`, `xxxxxx xxxxx6264`, + * `x xxxx 0887`) and is a presentation choice Frollo could change. All fourteen + * are distinct, as are the five excluded cards. + * + * Credit cards are absent on purpose — monthly statements already cover them and + * are the source of truth. Adding one here would create a second producer for + * rows the reconcile queue would then have to match against themselves. + */ +export const ACCOUNTS: AccountSpec[] = [ + { last4: "6264", label: "ANZ Access Advantage", provider: "ANZ" }, + { last4: "9940", label: "AMP (mortgage)", provider: "AMP" }, + { last4: "8109", label: "AMP", provider: "AMP" }, + { last4: "0682", label: "AMP (loan)", provider: "AMP" }, + { last4: "4830", label: "Wise (income)", provider: "Wise" }, + { last4: "2176", label: "Wise", provider: "Wise" }, + { last4: "0052", label: "Up Spending", provider: "Up" }, + { last4: "0078", label: "Up Savings", provider: "Up" }, + { last4: "1910", label: "CommBank Smart Access", provider: "CommBank" }, + { last4: "4878", label: "Westpac Choice", provider: "Westpac" }, + { last4: "4758", label: "Westpac Business One", provider: "Westpac" }, + { last4: "0887", label: "ING Orange Everyday", provider: "ING" }, + { last4: "9336", label: "ING Savings Maximiser", provider: "ING" }, + { last4: "9807", label: "ANZ Online Saver", provider: "ANZ" }, +]; + +/** + * Below this id gap, two rows sharing a natural key are *probably* a genuine + * same-day repeat rather than a re-ingest twin — and are reported rather than + * silently collapsed. + * + * This is a review trigger, not a decision. An earlier version used a 10,000 + * threshold to *keep* close-id rows, and it was wrong: measured across two real + * exports, in-scope duplicate pairs have gaps from **58 to 260 million**, while + * the only observed genuine repeats (five identical $1.00 Apple charges) had + * gaps of 1 to 4. A threshold anywhere in between is a magic number, and the + * error it enables is asymmetric — keeping a twin doubled reported income by + * 100%, whereas collapsing a real repeat understates by one row. + * + * So dedupe collapses unconditionally and flags anything this close for a human + * to look at. On the accounts in scope, nothing has ever tripped it: every + * duplicate group is a pair and the minimum gap is 58. + */ +export const SUSPECT_REPEAT_GAP = 10; + +export function last4(accountNumber: string): string { + return accountNumber.replace(/\D/g, "").slice(-4); +} + +export function accountFor(accountNumber: string): AccountSpec | undefined { + const key = last4(accountNumber); + return ACCOUNTS.find((a) => a.last4 === key); +} + +const REQUIRED_COLUMNS = [ + "transaction_id", + "description", + "amount", + "currency", + "transaction_date", + "account_number", + "account_name", + "credit_debit", +]; + +/** + * Maps a parsed CSV grid to rows, by header name rather than position. + * + * Throws on a missing required column rather than reading undefined into a + * transaction: a column rename upstream should stop the import loudly, not + * silently produce rows with blank descriptions. + */ +export function readRows(grid: string[][]): FrolloRow[] { + if (grid.length === 0) throw new Error("empty CSV"); + const header = grid[0].map((h) => h.trim()); + const missing = REQUIRED_COLUMNS.filter((c) => !header.includes(c)); + if (missing.length > 0) { + throw new Error(`Frollo CSV missing expected column(s): ${missing.join(", ")}`); + } + const at = (row: string[], col: string): string => { + const i = header.indexOf(col); + return i >= 0 ? (row[i] ?? "").trim() : ""; + }; + const out: FrolloRow[] = []; + for (const row of grid.slice(1)) { + if (row.every((f) => f === "")) continue; + const amount = Number(at(row, "amount")); + out.push({ + transactionId: at(row, "transaction_id"), + description: at(row, "description"), + amount: Number.isFinite(amount) ? amount : NaN, + currency: at(row, "currency") || "AUD", + transactionDate: at(row, "transaction_date"), + postedDate: at(row, "posted_date") || null, + accountNumber: at(row, "account_number"), + accountName: at(row, "account_name"), + creditDebit: at(row, "credit_debit"), + transactionType: at(row, "transaction_type") || null, + providerName: at(row, "provider_name"), + merchantName: at(row, "merchant_name") || null, + }); + } + return out; +} + +export type SkipReason = + | "zero_amount" + | "unparseable_amount" + | "out_of_scope_account" + | "pending"; + +/** + * Why a row is not imported, or null if it should be. + * + * `zero_amount` is mostly CommBank fee *waivers* — 574 of them in one export, + * genuinely $0.00 with the notional fee written into the description text + * (`INTNL TRANSACTION FEE, $ 41.58 FEE SAV`). They are not transactions. + * + * `pending` is a belt-and-braces check on top of the export option. It only + * catches Westpac's description prefix; Amex pending rows are undetectable here, + * which is why the option matters and this does not replace it. + */ +export function skipReason(row: FrolloRow): SkipReason | null { + if (!Number.isFinite(row.amount)) return "unparseable_amount"; + if (row.amount === 0) return "zero_amount"; + if (!accountFor(row.accountNumber)) return "out_of_scope_account"; + if (/^\s*pending\s*[-:]/i.test(row.description)) return "pending"; + return null; +} + +/** Natural key: what makes two rows the same purchase, ignoring ids. */ +export function naturalKey(row: FrolloRow): string { + return [ + last4(row.accountNumber), + row.transactionDate, + row.amount.toFixed(2), + row.description, + row.currency, + ].join(""); +} + +export interface DroppedRow { + row: FrolloRow; + duplicateOf: FrolloRow; + idGap: number; + /** Ids close enough that this might be a real repeat, not a twin. Review it. */ + suspectGenuineRepeat: boolean; +} + +export interface DedupeResult { + kept: FrolloRow[]; + dropped: DroppedRow[]; +} + +/** + * Collapses each natural-key group to its lowest-id row. + * + * Lowest because old ids survive a re-ingestion — verified across two exports + * twelve days apart, where the earlier ids were a strict subset of the later + * ones. Keeping the lowest therefore keeps `source_ref` stable from run to run, + * so a re-import updates nothing and inserts nothing. + * + * Everything collapsed is returned in `dropped`, and anything whose ids sit + * within SUSPECT_REPEAT_GAP is flagged: that is the shape a genuine same-day + * repeat would take, and it should be looked at rather than assumed. + */ +export function dedupe(rows: FrolloRow[], suspectGap = SUSPECT_REPEAT_GAP): DedupeResult { + const groups = new Map(); + for (const row of rows) { + const k = naturalKey(row); + const g = groups.get(k); + if (g) g.push(row); + else groups.set(k, [row]); + } + + const kept: FrolloRow[] = []; + const dropped: DroppedRow[] = []; + + for (const group of groups.values()) { + if (group.length === 1) { + kept.push(group[0]); + continue; + } + const sorted = [...group].sort( + (a, b) => Number(a.transactionId) - Number(b.transactionId) + ); + const winner = sorted[0]; + kept.push(winner); + for (let i = 1; i < sorted.length; i++) { + const idGap = Number(sorted[i].transactionId) - Number(sorted[i - 1].transactionId); + dropped.push({ + row: sorted[i], + duplicateOf: winner, + idGap, + suspectGenuineRepeat: idGap < suspectGap, + }); + } + } + + return { kept, dropped }; +} + +/** + * Shapes a row for `transactions`. + * + * `category` is deliberately not set. Frollo assigns its own categories and the + * owner reports they are often wrong; finance-app's rules engine plus + * `category_override` already decide this, and a third untrusted source + * competing with both is worse than none. + * + * For a non-AUD row the file gives only the foreign figure — there is no AUD + * equivalent anywhere in the export. That is the same position order-ingestion + * is in, so it takes the same shape: `amount` IS the native figure, + * `foreign_currency_code` names it, and `amount_aud` is left NULL rather than + * asserting a rate we do not have. `AMOUNT_UNCONVERTED` in analytics-sql then + * counts these and the UI reports the balance as incomplete. + */ +export function toLedgerRow(row: FrolloRow): LedgerRow { + const foreign = row.currency !== "AUD"; + const magnitude = Math.abs(row.amount); + return { + sourceRef: row.transactionId, + sourceAccount: row.accountNumber, + transactionDate: row.transactionDate, + description: row.description, + amount: magnitude, + transactionType: row.amount < 0 ? "debit" : "credit", + // Frollo writes "Unknown" where it could not resolve a merchant. Storing + // that would put the literal word on the screen; NULL lets the existing + // merchant fallbacks work. + merchantName: row.merchantName && row.merchantName !== "Unknown" ? row.merchantName : null, + foreignCurrencyAmount: foreign ? magnitude : null, + foreignCurrencyCode: foreign ? row.currency : null, + }; +} + +export interface AccountReport { + spec: AccountSpec; + rows: number; + present: boolean; +} + +/** + * Which configured accounts the file actually contains. + * + * A lapsed CDR consent does not error — the account simply stops appearing, and + * the row count drops. Nothing else in the pipeline would notice, so the import + * asserts the roster rather than trusting it. + */ +export function accountReport(rows: FrolloRow[]): AccountReport[] { + const counts = new Map(); + for (const r of rows) { + const k = last4(r.accountNumber); + counts.set(k, (counts.get(k) ?? 0) + 1); + } + return ACCOUNTS.map((spec) => ({ + spec, + rows: counts.get(spec.last4) ?? 0, + present: (counts.get(spec.last4) ?? 0) > 0, + })); +} + +/** Accounts in the file that this importer does not know about. */ +export function unknownAccounts(rows: FrolloRow[]): { accountNumber: string; accountName: string; rows: number }[] { + const seen = new Map(); + for (const r of rows) { + if (accountFor(r.accountNumber)) continue; + const k = last4(r.accountNumber); + const e = seen.get(k); + if (e) e.rows++; + else seen.set(k, { accountNumber: r.accountNumber, accountName: r.accountName, rows: 1 }); + } + return [...seen.values()].sort((a, b) => b.rows - a.rows); +} diff --git a/src/lib/queries.ts b/src/lib/queries.ts index ac98c4e..8608cf3 100644 --- a/src/lib/queries.ts +++ b/src/lib/queries.ts @@ -732,6 +732,25 @@ export async function batchInsertCSVTransactions( export const needsCardMatch = (alias = "t") => `(${alias}.payment_method IS NULL OR ${alias}.payment_method NOT IN ('cash', 'credits'))`; +/** + * Rows for which a statement line could still arrive. + * + * A row imported from an account feed is that account's own ledger entry, not a + * receipt waiting to be matched against one. The Frollo importer deliberately + * covers only accounts whose statements are *not* imported — credit cards are + * excluded from it precisely because theirs are — so no statement line is coming + * for these, ever. Same shape as a credits-funded order, different reason. + * + * Without this, roughly 600 rows a year would sit in the pending-reconciliation + * queue forever and bury the receipts that genuinely need a decision. Scoped to + * feeds rather than to `source IS NOT NULL`, so a future source that *does* await + * a statement is not silently swept up by it. + */ +export const ACCOUNT_FEED_SOURCES = ["frollo"] as const; + +export const awaitsStatementLine = (alias = "t") => + `(${alias}.source IS NULL OR ${alias}.source NOT IN (${ACCOUNT_FEED_SOURCES.map((s) => `'${s}'`).join(", ")}))`; + /** * Bank label for a transaction. A row with no statement was not imported from * one, and the label has to say *why*: "Manual" reads as "hand-entered, still @@ -784,6 +803,7 @@ export async function getPendingReconciliations(ownerId: number): Promise