diff --git a/scripts/import-frollo.mts b/scripts/import-frollo.mts deleted file mode 100644 index a0693bb..0000000 --- a/scripts/import-frollo.mts +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Imports a Frollo transaction export into `transactions`. - * - * node --experimental-strip-types scripts/import-frollo.mts --file - * node --experimental-strip-types scripts/import-frollo.mts --file --apply - * - * Dry run by default: it prints exactly what it would do and touches nothing. - * Rehearse first — the failure mode here is doubling reported income, and the - * de-duplication rule that shipped second only survived because a dry run was - * read against the salary rows. - * - * Export the file with PENDING TRANSACTIONS EXCLUDED. A pending row changes both - * its id and its description when it settles, so importing one guarantees a - * duplicate on the next run; every pending row observed has been on a credit - * card, which this importer does not cover anyway. See DECISIONS.md ING-11. - * - * All parsing, scoping, de-duplication and insertion live in - * `src/lib/frollo-ingest.ts`, shared with POST /api/frollo/ingest so the manual - * and automatic paths cannot drift. - * - * Needs DATABASE_URL. postgres-personal publishes no host port, so from the host: - * export DATABASE_URL="postgresql://personal:@$(docker inspect postgres-personal \ - * --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):5432/personal" - * The container IP changes on every recreate. - */ -import { readFileSync } from "node:fs"; -import pg from "pg"; -import { ingestFrolloCsv, LEDGER_MATCH_DAYS, type SqlExecutor } from "../src/lib/frollo-ingest.ts"; - -function arg(name: string, fallback?: string): string | undefined { - const i = process.argv.indexOf(`--${name}`); - if (i >= 0 && process.argv[i + 1] && !process.argv[i + 1].startsWith("--")) return process.argv[i + 1]; - return fallback; -} -const has = (name: string) => process.argv.includes(`--${name}`); - -const file = arg("file"); -if (!file) { - console.error("usage: --file [--apply] [--force] [--owner ]"); - process.exit(2); -} -if (!process.env.DATABASE_URL) { - console.error("DATABASE_URL is not set — cannot compare against the ledger."); - process.exit(2); -} - -const money = (n: number) => n.toLocaleString("en-AU", { minimumFractionDigits: 2, maximumFractionDigits: 2 }); - -const client = new pg.Client({ connectionString: process.env.DATABASE_URL }); -await client.connect(); -const exec: SqlExecutor = async (sql: string, params: unknown[]) => - (await client.query(sql, params)).rows as T[]; - -let report; -try { - report = await ingestFrolloCsv(readFileSync(file, "utf8"), exec, { - ownerId: Number(arg("owner", "1")), - apply: has("apply"), - force: has("force"), - }); -} finally { - // Left open, the process hangs after an exception and looks like a slow query. - if (!has("keep-open")) await client.end().catch(() => {}); -} - -console.log(`\n${file}`); -console.log(` ${report.totalRows} rows in file\n`); - -console.log("SCOPE"); -for (const [why, n] of Object.entries(report.skipped).sort((a, b) => b[1]! - a[1]!)) { - console.log(` skipped ${String(n).padStart(5)} ${why}`); -} -console.log(` in scope ${String(report.inScope).padStart(4)}\n`); - -console.log("ACCOUNTS"); -for (const a of report.accounts) { - console.log( - ` ${a.present ? " " : "!"} ${a.spec.last4} ${a.spec.label.padEnd(24)} ${String(a.rows).padStart(5)} rows` - ); -} -if (report.unknownAccounts.length > 0) { - console.log("\n accounts in the file this importer does not know (skipped):"); - for (const u of report.unknownAccounts) { - console.log(` ${u.accountNumber} ${u.accountName} ${u.rows} rows`); - } -} -console.log(); - -console.log("DE-DUPLICATION (re-ingest twins from CDR re-consent)"); -console.log(` ${report.duplicatesDropped} dropped`); -if (report.suspectRepeats.length > 0) { - console.log(`\n REVIEW: ${report.suspectRepeats.length} collapsed row(s) had near-consecutive ids`); - console.log(" and may be real repeats rather than duplicates:"); - for (const s of report.suspectRepeats.slice(0, 10)) { - console.log(` ${s.transactionDate} ${money(s.amount).padStart(11)} ${s.description.slice(0, 40)} (gap ${s.idGap})`); - } -} -console.log(); - -console.log("LEDGER"); -console.log(` already imported : ${String(report.alreadyImported).padStart(5)}`); -// Printed even when zero. These are the counts that were missing on -// 2026-08-13, when the whole file was written on top of a ledger that already -// held 77% of it; a number you have to go looking for is a number nobody looks -// at. The watermarks are printed too, so a wrong coverage boundary is visible -// rather than inferred from a row count. -console.log(` covered by stmt : ${String(report.coveredByStatement).padStart(5)} (dated on or before the account's newest statement)`); -console.log(` amount twin : ${String(report.ledgerDuplicates).padStart(5)} (same amount + direction within ${LEDGER_MATCH_DAYS} days)`); -if (report.statementWatermarks.length > 0) { - console.log(" statements cover:"); - for (const w of report.statementWatermarks) console.log(` ${w.last4} up to ${w.coveredTo}`); -} -console.log(` new to insert : ${String(report.toInsert).padStart(5)}`); - -if (report.anomalies.length > 0) { - console.log("\nANOMALIES"); - for (const a of report.anomalies) console.log(` ! ${a}`); -} - -if (report.applied) { - console.log(`\nInserted ${report.inserted} row(s).\n`); -} else if (report.anomalies.length > 0 && has("apply")) { - console.log("\nNOT APPLIED — anomalies above. Read them, then re-run with --force.\n"); - process.exit(1); -} else { - console.log("\nDRY RUN — nothing written. Re-run with --apply to insert.\n"); -} diff --git a/src/__tests__/unit/csv-import-coverage.test.ts b/src/__tests__/unit/csv-import-coverage.test.ts new file mode 100644 index 0000000..ae57901 --- /dev/null +++ b/src/__tests__/unit/csv-import-coverage.test.ts @@ -0,0 +1,183 @@ +import { describe, it, expect } from "vitest"; +import { + applyMapping, + inFileDuplicates, + last4, + splitByCoverage, + type ColumnMapping, + type ParsedTransaction, +} from "@/lib/csv-parser"; + +/** + * The statement-coverage rule, which replaced a whole Frollo-specific importer. + * + * The first aggregator import wrote 422 duplicate rows out of 550 — $1,023,824 + * of movement counted twice — because "these accounts issue no statements" was + * asserted rather than queried. Amount-matching could not fix it either: the two + * sources decompose the same event differently, bundling a transfer fee where + * the statement itemises it. + * + * A statement's end date answers the question that actually has an answer: up + * to what date is this account complete? On the real 2026-08-13 export it takes + * 2,607 rows down to 171, with no account allowlist and no card exclusions. + */ + +const tx = (o: Partial & { date: string }): ParsedTransaction => ({ + description: "SOMETHING", + amount: 10, + transaction_type: "debit", + ...o, +}); + +describe("last4", () => { + it("reads the same account written four different ways", () => { + // Every one of these forms appears in the real data. + expect(last4("xxxxxxxxxxxx2176")).toBe("2176"); + expect(last4("235242176")).toBe("2176"); + expect(last4("xxx-xxx xx4878")).toBe("4878"); + expect(last4("4085-56264")).toBe("6264"); + }); + + it("returns empty for an unusable identifier rather than guessing", () => { + expect(last4("N/A")).toBe(""); + expect(last4("12")).toBe(""); + expect(last4("")).toBe(""); + }); +}); + +describe("splitByCoverage", () => { + const coverage = [{ last4: "2176", coveredTo: "2026-07-26" }]; + + it("excludes a row on or before the account's newest statement", () => { + const { keep, covered } = splitByCoverage( + [tx({ date: "2026-07-25", account: "xxxxxxxxxxxx2176" })], + coverage + ); + expect(covered).toHaveLength(1); + expect(keep).toHaveLength(0); + }); + + it("treats the watermark date itself as covered", () => { + const { covered } = splitByCoverage( + [tx({ date: "2026-07-26", account: "xxxxxxxxxxxx2176" })], + coverage + ); + expect(covered).toHaveLength(1); + }); + + it("keeps a row after the watermark — the whole point of the feed", () => { + // The 2026-08-12 salary: the statement has not arrived, so this row is the + // only record of it and must survive. + const { keep } = splitByCoverage( + [tx({ date: "2026-08-12", account: "xxxxxxxxxxxx2176", amount: 10782 })], + coverage + ); + expect(keep).toHaveLength(1); + }); + + it("keeps everything for an account with no statements", () => { + const { keep, covered } = splitByCoverage( + [tx({ date: "2020-01-01", account: "xxx-xxx xx4878" })], + coverage + ); + expect(keep).toHaveLength(1); + expect(covered).toHaveLength(0); + }); + + it("keeps everything when no account column was mapped", () => { + // A single-account bank export has no account column, and a row must never + // be dropped on a guess about which account it belongs to. + const { keep, covered } = splitByCoverage([tx({ date: "2020-01-01" })], coverage); + expect(keep).toHaveLength(1); + expect(covered).toHaveLength(0); + }); + + it("applies each account's own watermark, not a global one", () => { + // The real export spans watermarks from 2026-03-31 to 2026-07-26. A single + // date would either import duplicates or discard real rows. + const { keep, covered } = splitByCoverage( + [ + tx({ date: "2026-05-01", account: "xxxx0887" }), // covered to 2026-03-31 → keep + tx({ date: "2026-05-01", account: "xxxx2176" }), // covered to 2026-07-26 → drop + ], + [ + { last4: "0887", coveredTo: "2026-03-31" }, + { last4: "2176", coveredTo: "2026-07-26" }, + ] + ); + expect(keep).toHaveLength(1); + expect(keep[0].account).toBe("xxxx0887"); + expect(covered).toHaveLength(1); + }); +}); + +describe("inFileDuplicates", () => { + it("counts twins that a row id cannot catch", () => { + // A CDR re-consent re-exports an account's whole history under fresh ids + // while the originals survive, so source_ref sees two distinct rows. 385 of + // these were in one 2,563-row export and doubled every salary payment. + const rows = [ + tx({ date: "2026-07-07", amount: 10782, description: "HDR", account: "4830" }), + tx({ date: "2026-07-07", amount: 10782, description: "HDR", account: "4830" }), + ]; + expect(inFileDuplicates(rows)).toBe(1); + }); + + it("does not flag the same amount on different days", () => { + expect( + inFileDuplicates([ + tx({ date: "2026-07-07", amount: 5, description: "COFFEE" }), + tx({ date: "2026-07-08", amount: 5, description: "COFFEE" }), + ]) + ).toBe(0); + }); + + it("does not flag the same amount on different accounts", () => { + // Both legs of an internal transfer: same day, same amount, different + // accounts, and both are real. + expect( + inFileDuplicates([ + tx({ date: "2026-05-18", amount: 3076.04, description: "Transfer", account: "6264" }), + tx({ date: "2026-05-18", amount: 3076.04, description: "Transfer", account: "9940" }), + ]) + ).toBe(0); + }); +}); + +describe("applyMapping with account and row-id columns", () => { + const labels = ["transaction_id", "description", "amount", "transaction_date", "account_number"]; + const mapping: ColumnMapping = { + dateCol: "transaction_date", + descriptionCol: "description", + amountMode: "single", + amountCol: "amount", + accountCol: "account_number", + sourceRefCol: "transaction_id", + }; + + it("carries the account and row id through", () => { + const out = applyMapping( + [["1326131276", "HDR Global Services", "10782.00", "2026-08-12", "xxxxxxxxxxxx4830"]], + labels, + mapping, + "YYYY-MM-DD" + ); + expect(out).toHaveLength(1); + expect(out[0].account).toBe("xxxxxxxxxxxx4830"); + expect(out[0].source_ref).toBe("1326131276"); + // A positive single-column amount is money in. + expect(out[0].transaction_type).toBe("credit"); + }); + + it("leaves both undefined when the columns are not mapped", () => { + const out = applyMapping( + [["1", "SOMETHING", "-42.50", "2026-08-12", "xxxx4830"]], + labels, + { dateCol: "transaction_date", descriptionCol: "description", amountMode: "single", amountCol: "amount" }, + "YYYY-MM-DD" + ); + expect(out[0].account).toBeUndefined(); + expect(out[0].source_ref).toBeUndefined(); + expect(out[0].transaction_type).toBe("debit"); + }); +}); diff --git a/src/__tests__/unit/frollo-csv.test.ts b/src/__tests__/unit/frollo-csv.test.ts deleted file mode 100644 index b15775b..0000000 --- a/src/__tests__/unit/frollo-csv.test.ts +++ /dev/null @@ -1,388 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { parseCSVRows } from "@/lib/csv-parser"; -import { - ACCOUNTS, - EXCLUDED_ACCOUNTS, - isKnownExcluded, - SUSPECT_REPEAT_GAP, - accountFor, - accountReport, - dedupe, - last4, - naturalKey, - readRows, - skipReason, - toLedgerRow, - unknownAccounts, - type FrolloRow, -} from "@/lib/frollo-csv"; -import { LARGE_BATCH, findAnomalies } from "@/lib/frollo-ingest"; - -const HEADER = - "transaction_id,description,user_description,amount,currency,transaction_date,posted_date," + - "account_number,account_name,credit_debit,transaction_type,provider_name,merchant_name," + - "budget_category,category_name,user_tags,notes,included"; - -function csv(...lines: string[]): FrolloRow[] { - return readRows(parseCSVRows([HEADER, ...lines].join("\n"))); -} - -/** Shapes a line in the export's real column order. */ -function line(o: { - id: string; - desc?: string; - amount: string; - currency?: string; - date?: string; - posted?: string; - account?: string; - accountName?: string; - cd?: string; - type?: string; - merchant?: string; -}): string { - return [ - o.id, - o.desc ?? "SOME MERCHANT", - "", - o.amount, - o.currency ?? "AUD", - o.date ?? "2026-03-01", - o.posted ?? "", - o.account ?? "xxx-xxx xxxx1910", - o.accountName ?? "Smart Access", - o.cd ?? (o.amount.startsWith("-") ? "debit" : "credit"), - o.type ?? "payment", - "CommBank", - o.merchant ?? "Some Merchant", - "lifestyle", - "Groceries", - '""', - "", - "true", - ].join(","); -} - -describe("readRows", () => { - it("maps by header name, not position", () => { - const rows = csv(line({ id: "1", amount: "-12.34", desc: "COFFEE" })); - expect(rows).toHaveLength(1); - expect(rows[0]).toMatchObject({ - transactionId: "1", - description: "COFFEE", - amount: -12.34, - currency: "AUD", - accountName: "Smart Access", - }); - }); - - it("throws when a required column is missing rather than reading undefined", () => { - const broken = "transaction_id,description,amount\n1,X,-1.00"; - expect(() => readRows(parseCSVRows(broken))).toThrow(/missing expected column/i); - }); - - it("keeps a quoted description containing commas intact", () => { - const desc = '"DIDI_NZ DidiNZ pending Auckland NZL, ##0626 8.77 NZ DOLLAR"'; - const rows = csv(line({ id: "1", amount: "-7.23", desc })); - expect(rows[0].description).toContain("NZL, ##0626"); - }); - - it("treats an absent posted_date as null, not empty string", () => { - expect(csv(line({ id: "1", amount: "-1.00" }))[0].postedDate).toBeNull(); - }); -}); - -describe("account scoping", () => { - it("matches on the last four digits regardless of masking format", () => { - expect(last4("xxx-xxx xxxx1910")).toBe("1910"); - expect(last4("xxxxxx xxxxx6264")).toBe("6264"); - expect(last4("x xxxx 0887 ")).toBe("0887"); - expect(last4("XXXXXXXXXXXX0351")).toBe("0351"); - }); - - it("has no duplicate last4 among configured accounts", () => { - const keys = ACCOUNTS.map((a) => a.last4); - expect(new Set(keys).size).toBe(keys.length); - }); - - it("excludes the credit cards, which statements already cover", () => { - for (const card of ["3893", "8032", "1000", "0351", "6227"]) { - expect(ACCOUNTS.find((a) => a.last4 === card)).toBeUndefined(); - } - }); - - it("recognises the loan and income accounts", () => { - expect(accountFor("xxx-xxx xxxxx0682")?.label).toMatch(/loan/i); - expect(accountFor("xxxxxxxxxxxx4830")?.label).toMatch(/income/i); - }); -}); - -describe("skipReason", () => { - it("skips a zero-amount fee waiver", () => { - const row = csv( - line({ id: "1", amount: "0.0", desc: "INTNL TRANSACTION FEE, $ 41.58 FEE SAV" }) - )[0]; - expect(skipReason(row)).toBe("zero_amount"); - }); - - it("skips a credit-card row", () => { - const row = csv(line({ id: "1", amount: "-50.00", account: "xxxx xxxx xxxx 3893" }))[0]; - expect(skipReason(row)).toBe("out_of_scope_account"); - }); - - it("skips a Westpac pending row on its description prefix", () => { - const row = csv( - line({ id: "1", amount: "-13.50", desc: "Pending - Breadworld Bakery Cafe", account: "xxx-xxx xx4878" }) - )[0]; - expect(skipReason(row)).toBe("pending"); - }); - - it("accepts an ordinary in-scope row", () => { - expect(skipReason(csv(line({ id: "1", amount: "-29.17" }))[0])).toBeNull(); - }); - - it("skips rather than imports an unparseable amount", () => { - const row = csv(line({ id: "1", amount: "n/a" }))[0]; - expect(skipReason(row)).toBe("unparseable_amount"); - }); -}); - -describe("dedupe", () => { - it("collapses a re-ingest twin to the earlier id", () => { - const rows = csv( - line({ id: "1200000000", amount: "-61.03", desc: "AMAZON MARKETPLACE" }), - line({ id: "1326000000", amount: "-61.03", desc: "AMAZON MARKETPLACE" }) - ); - const { kept, dropped } = dedupe(rows); - expect(kept).toHaveLength(1); - expect(kept[0].transactionId).toBe("1200000000"); - expect(dropped).toHaveLength(1); - expect(dropped[0].duplicateOf.transactionId).toBe("1200000000"); - }); - - it("collapses close-id repeats too, but flags them for review", () => { - // Five identical $1.00 Apple charges with consecutive ids. On a card - // account, so out of scope in practice — but if such a group ever reaches - // dedupe it must be surfaced, not silently decided either way. - const rows = csv( - ...["1326128755", "1326128756", "1326128757", "1326128758", "1326128759"].map((id) => - line({ id, amount: "-1.0", desc: "APPLE.COM/BILL SYDNEY", date: "2026-07-28" }) - ) - ); - const { kept, dropped } = dedupe(rows); - expect(kept).toHaveLength(1); - expect(dropped).toHaveLength(4); - expect(dropped.every((d) => d.suspectGenuineRepeat)).toBe(true); - }); - - it("does not flag a far-apart twin as a possible repeat", () => { - const rows = csv( - line({ id: "1101410977", amount: "10790.8", currency: "USD", account: "xxxxxxxxxxxx4830" }), - line({ id: "1101415475", amount: "10790.8", currency: "USD", account: "xxxxxxxxxxxx4830" }) - ); - const { kept, dropped } = dedupe(rows); - expect(kept).toHaveLength(1); - expect(dropped[0].suspectGenuineRepeat).toBe(false); - expect(dropped[0].idGap).toBe(4498); - }); - - it("collapses a real income twin — the 100%-overstatement case", () => { - // Every HDR salary payment appears twice in every export, id gaps ranging - // from 58 to 79 million. Keeping both doubled reported income. - const rows = csv( - line({ id: "1126624097", amount: "10790.8", currency: "USD", account: "xxxxxxxxxxxx4830", desc: "HDR Global Services" }), - line({ id: "1126624242", amount: "10790.8", currency: "USD", account: "xxxxxxxxxxxx4830", desc: "HDR Global Services" }) - ); - const { kept } = dedupe(rows); - expect(kept).toHaveLength(1); - expect(kept[0].transactionId).toBe("1126624097"); - }); - - it("does not merge rows that differ in amount, date, description or account", () => { - const rows = csv( - line({ id: "1000000001", amount: "-10.00" }), - line({ id: "1326000001", amount: "-10.01" }), - line({ id: "1326000002", amount: "-10.00", date: "2026-03-02" }), - line({ id: "1326000003", amount: "-10.00", desc: "OTHER" }), - line({ id: "1326000004", amount: "-10.00", account: "xxx-xxx xx4878" }) - ); - expect(dedupe(rows).kept).toHaveLength(5); - }); - - it("keeps the lowest id, because old ids survive a re-ingest", () => { - const rows = csv( - line({ id: "1326000001", amount: "-5.00" }), - line({ id: "1000000000", amount: "-5.00" }), - line({ id: "1200000000", amount: "-5.00" }) - ); - const { kept } = dedupe(rows); - expect(kept).toHaveLength(1); - expect(kept[0].transactionId).toBe("1000000000"); - }); - - it("reports the id gap so a suspect group can be reviewed", () => { - const near = csv( - line({ id: "1000000000", amount: "-5.00" }), - line({ id: String(1000000000 + SUSPECT_REPEAT_GAP - 1), amount: "-5.00" }) - ); - expect(dedupe(near).dropped[0].suspectGenuineRepeat).toBe(true); - const far = csv( - line({ id: "1000000000", amount: "-5.00" }), - line({ id: String(1000000000 + SUSPECT_REPEAT_GAP), amount: "-5.00" }) - ); - expect(dedupe(far).dropped[0].suspectGenuineRepeat).toBe(false); - }); - - it("is idempotent", () => { - const rows = csv( - line({ id: "1000000001", amount: "-61.03" }), - line({ id: "1326000001", amount: "-61.03" }) - ); - const once = dedupe(rows).kept; - expect(dedupe(once).kept).toEqual(once); - }); -}); - -describe("naturalKey", () => { - it("is stable across masking differences on the same account", () => { - const a = csv(line({ id: "1", amount: "-1.00", account: "xxx-xxx xxxx1910" }))[0]; - const b = csv(line({ id: "2", amount: "-1.00", account: "xx-xxx xxxxxx1910" }))[0]; - expect(naturalKey(a)).toBe(naturalKey(b)); - }); -}); - -describe("toLedgerRow", () => { - it("stores a debit as a positive amount with a direction", () => { - const r = toLedgerRow(csv(line({ id: "1", amount: "-29.17" }))[0]); - expect(r.amount).toBe(29.17); - expect(r.transactionType).toBe("debit"); - }); - - it("marks a positive amount as a credit", () => { - const r = toLedgerRow(csv(line({ id: "1", amount: "302.70" }))[0]); - expect(r.amount).toBe(302.7); - expect(r.transactionType).toBe("credit"); - }); - - it("leaves a foreign row's amount native and names the currency", () => { - const r = toLedgerRow( - csv(line({ id: "1", amount: "-10001.13", currency: "USD", account: "xxxxxxxxxxxx4830" }))[0] - ); - expect(r.amount).toBe(10001.13); - expect(r.foreignCurrencyCode).toBe("USD"); - expect(r.foreignCurrencyAmount).toBe(10001.13); - }); - - it("carries no foreign fields for an AUD row", () => { - const r = toLedgerRow(csv(line({ id: "1", amount: "-5.00" }))[0]); - expect(r.foreignCurrencyCode).toBeNull(); - expect(r.foreignCurrencyAmount).toBeNull(); - }); - - it("drops Frollo's literal 'Unknown' merchant rather than showing it", () => { - const r = toLedgerRow(csv(line({ id: "1", amount: "-5.00", merchant: "Unknown" }))[0]); - expect(r.merchantName).toBeNull(); - }); - - it("keeps a real merchant", () => { - const r = toLedgerRow(csv(line({ id: "1", amount: "-5.00", merchant: "Solar Victoria" }))[0]); - expect(r.merchantName).toBe("Solar Victoria"); - }); - - it("carries the provider's id as source_ref for idempotency", () => { - const r = toLedgerRow(csv(line({ id: "986846710", amount: "-5.00" }))[0]); - expect(r.sourceRef).toBe("986846710"); - }); -}); - -describe("accountReport / unknownAccounts", () => { - it("reports a configured account with no rows as absent — a lapsed consent", () => { - const rows = csv(line({ id: "1", amount: "-5.00", account: "xxx-xxx xxxx1910" })); - const report = accountReport(rows); - expect(report.find((r) => r.spec.last4 === "1910")?.present).toBe(true); - expect(report.find((r) => r.spec.last4 === "4830")?.present).toBe(false); - }); - - it("surfaces an account in the file that the importer does not know", () => { - const rows = csv(line({ id: "1", amount: "-5.00", account: "xxxx xxxx xxxx 9999", accountName: "New Card" })); - const unknown = unknownAccounts(rows); - expect(unknown).toHaveLength(1); - expect(unknown[0]).toMatchObject({ accountName: "New Card", rows: 1 }); - }); -}); - -describe("findAnomalies", () => { - const base = { missingAccounts: [], unknownAccounts: [], suspectRepeats: [], toInsert: 10, skipped: {} }; - - it("passes a boring run", () => { - expect(findAnomalies(base)).toEqual([]); - }); - - it("flags a configured account that contributed nothing — a lapsed consent", () => { - const out = findAnomalies({ ...base, missingAccounts: ["4830 Wise (income)"] }); - expect(out).toHaveLength(1); - expect(out[0]).toMatch(/consent may have lapsed/i); - }); - - it("flags an account that appeared in the file unannounced", () => { - const out = findAnomalies({ - ...base, - unknownAccounts: [{ accountNumber: "xxxx9999", accountName: "New Everyday", rows: 4 }], - }); - expect(out[0]).toMatch(/unknown account/i); - }); - - it("flags a de-duplication that might have collapsed a real repeat", () => { - const out = findAnomalies({ - ...base, - suspectRepeats: [{ description: "X", transactionDate: "2026-01-01", amount: -1, idGap: 2 }], - }); - expect(out[0]).toMatch(/genuine repeats/i); - }); - - it("flags a batch far larger than an ordinary month", () => { - expect(findAnomalies({ ...base, toInsert: LARGE_BATCH + 1 })).toHaveLength(1); - expect(findAnomalies({ ...base, toInsert: LARGE_BATCH })).toHaveLength(0); - }); - - it("flags an export taken with pending included", () => { - const out = findAnomalies({ ...base, skipped: { pending: 3 } }); - expect(out[0]).toMatch(/pending/i); - }); - - it("reports every anomaly at once rather than the first", () => { - expect( - findAnomalies({ ...base, missingAccounts: ["a"], toInsert: LARGE_BATCH + 1, skipped: { pending: 1 } }) - ).toHaveLength(3); - }); -}); - -describe("excluded vs unknown accounts", () => { - it("does not report a deliberately excluded credit card as unknown", () => { - // An alert that fires on every run is an alert nobody reads. - const rows = csv(line({ id: "1", amount: "-50.00", account: "xxxx xxxx xxxx 3893", accountName: "Ultimate Awards credit card" })); - expect(unknownAccounts(rows)).toEqual([]); - expect(isKnownExcluded("xxxx xxxx xxxx 3893")).toBe(true); - }); - - it("still reports a genuinely new account", () => { - const rows = csv(line({ id: "1", amount: "-50.00", account: "xxxx xxxx xxxx 9999", accountName: "Brand New" })); - expect(unknownAccounts(rows)).toHaveLength(1); - }); - - it("keeps the imported and excluded rosters disjoint", () => { - const a = new Set(ACCOUNTS.map((x) => x.last4)); - for (const e of EXCLUDED_ACCOUNTS) expect(a.has(e.last4)).toBe(false); - }); -}); - -describe("pending detection precedes account scoping", () => { - it("reports a pending credit-card row as pending, not out of scope", () => { - // Otherwise the wrong-export-option signal never fires, since every pending - // row observed so far has been on a card. - const row = csv( - line({ id: "1", amount: "-13.50", desc: "Pending - Breadworld Bakery Cafe", account: "xxxx xxxx xxxx 8032" }) - )[0]; - expect(skipReason(row)).toBe("pending"); - }); -}); diff --git a/src/__tests__/unit/frollo-ingest.test.ts b/src/__tests__/unit/frollo-ingest.test.ts deleted file mode 100644 index 32ef25b..0000000 --- a/src/__tests__/unit/frollo-ingest.test.ts +++ /dev/null @@ -1,229 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { ingestFrolloCsv, LEDGER_MATCH_DAYS, type SqlExecutor } from "@/lib/frollo-ingest"; - -/** - * Tests for the ledger-duplicate guard. - * - * This is the check whose absence caused the 2026-08-13 first import to write - * 422 duplicate rows out of 550 — every one of them a second copy of a payment - * the ledger already held from a statement. It passed 29 unit tests and a clean - * dry run at the time, because every one of those tests compared the CSV against - * itself. Nothing compared it against the database. - */ - -const HEADER = - "transaction_id,description,user_description,amount,currency,transaction_date,posted_date," + - "account_number,account_name,credit_debit,transaction_type,provider_name,merchant_name," + - "budget_category,category_name,user_tags,notes,included"; - -function line(o: { id: string; desc?: string; amount: string; date: string }): string { - return [ - o.id, - o.desc ?? "SOME MERCHANT", - "", - o.amount, - "AUD", - o.date, - "", - "xxx-xxx xxxx1910", - "Smart Access", - o.amount.startsWith("-") ? "debit" : "credit", - "payment", - "CommBank", - "Some Merchant", - "lifestyle", - "Groceries", - "", - "", - "true", - ].join(","); -} - -const csv = (...lines: string[]) => [HEADER, ...lines].join("\n"); - -/** - * Stands in for the database. The two queries are told apart by their text, - * which is crude but keeps the test free of a live connection — and the guard - * is pure logic over what comes back. - */ -function exec( - prior: { transaction_date: string; amount: string; transaction_type?: string }[], - coverage: { last4: string; covered_to: string }[] = [] -): SqlExecutor { - return (async (sql: string) => { - if (sql.includes("WHERE source = $1")) return []; - if (sql.includes("FROM statements")) return coverage; - if (sql.includes("superseded_by_id IS NULL")) return prior.map((p) => ({ transaction_type: "debit", ...p })); - return []; - }) as SqlExecutor; -} - -describe("ledger-duplicate guard", () => { - it("drops a row the ledger already holds on the same day", async () => { - const r = await ingestFrolloCsv( - csv(line({ id: "1", amount: "-42.50", date: "2026-03-10" })), - exec([{ transaction_date: "2026-03-10", amount: "42.50" }]) - ); - expect(r.ledgerDuplicates).toBe(1); - expect(r.toInsert).toBe(0); - }); - - it("drops a row dated within the match window", async () => { - // The feed dates a transaction when the provider posted it and a statement - // when the bank did; a weekend puts two days between them. - const r = await ingestFrolloCsv( - csv(line({ id: "1", amount: "-42.50", date: "2026-03-10" })), - exec([{ transaction_date: "2026-03-13", amount: "42.50" }]) - ); - expect(LEDGER_MATCH_DAYS).toBe(3); - expect(r.ledgerDuplicates).toBe(1); - expect(r.toInsert).toBe(0); - }); - - it("keeps a row beyond the match window", async () => { - const r = await ingestFrolloCsv( - csv(line({ id: "1", amount: "-42.50", date: "2026-03-10" })), - exec([{ transaction_date: "2026-03-14", amount: "42.50" }]) - ); - expect(r.ledgerDuplicates).toBe(0); - expect(r.toInsert).toBe(1); - }); - - it("keeps a row whose amount differs", async () => { - const r = await ingestFrolloCsv( - csv(line({ id: "1", amount: "-42.50", date: "2026-03-10" })), - exec([{ transaction_date: "2026-03-10", amount: "42.51" }]) - ); - expect(r.ledgerDuplicates).toBe(0); - expect(r.toInsert).toBe(1); - }); - - it("consumes each ledger row once, so a genuine repeat survives", async () => { - // Two real charges of the same amount in the same week, one of which the - // ledger already has. Matching without consuming would drop both and lose a - // transaction that never existed anywhere else. - const r = await ingestFrolloCsv( - csv( - line({ id: "1", desc: "COFFEE ONE", amount: "-5.00", date: "2026-03-10" }), - line({ id: "2", desc: "COFFEE TWO", amount: "-5.00", date: "2026-03-11" }) - ), - exec([{ transaction_date: "2026-03-10", amount: "5.00" }]) - ); - expect(r.ledgerDuplicates).toBe(1); - expect(r.toInsert).toBe(1); - }); - - it("matches when the driver returns Date objects, not strings", async () => { - // `pg` maps a Postgres DATE to a JS Date; Prisma and the CSV give strings. - // The first cut of this guard read one as the other, and because the failure - // is a silent NaN it reported 550 rows to insert against a ledger holding - // 422 of them. Both shapes are tested because both drivers are in use: the - // CLI runs on pg, the API route on Prisma. - const asDate = await ingestFrolloCsv( - csv(line({ id: "1", amount: "-42.50", date: "2026-03-10" })), - exec([{ transaction_date: new Date(2026, 2, 10) as unknown as string, amount: "42.50" }]) - ); - expect(asDate.ledgerDuplicates).toBe(1); - expect(asDate.toInsert).toBe(0); - }); - - it("never matches a credit against a debit", async () => { - // Internal transfers between the owner's own accounts put both legs in the - // feed: 2026-05-18 carries +3076.04 into ANZ and -3076.04 out of AMP. On - // amount alone the credit leg consumed the ledger's debit row and the real - // duplicate was written. 20 rows got in this way before direction was part - // of the key. - const r = await ingestFrolloCsv( - csv( - line({ id: "1", desc: "PAYMENT FROM SELF", amount: "3076.04", date: "2026-05-18" }), - line({ id: "2", desc: "Transfer to Self", amount: "-3076.04", date: "2026-05-18" }) - ), - exec([{ transaction_date: "2026-05-18", amount: "3076.04", transaction_type: "debit" }]) - ); - // The debit leg is the duplicate; the credit leg is a genuinely new row. - expect(r.ledgerDuplicates).toBe(1); - expect(r.toInsert).toBe(1); - }); - - it("treats a statement 'refund' as money in", async () => { - // The feed calls a reversed account fee a credit; the statement importer - // types it 'refund'. Classifying refund as an outflow left every ANZ - // servicing-fee reversal in the file as a duplicate. - const r = await ingestFrolloCsv( - csv(line({ id: "1", desc: "REVERSAL OF ACCOUNT SERVICING FEE", amount: "5.00", date: "2026-02-27" })), - exec([{ transaction_date: "2026-02-27", amount: "5.00", transaction_type: "refund" }]) - ); - expect(r.ledgerDuplicates).toBe(1); - expect(r.toInsert).toBe(0); - }); - - it("treats a statement 'fee' as money out", async () => { - const r = await ingestFrolloCsv( - csv(line({ id: "1", desc: "ACCOUNT SERVICING FEE", amount: "-5.00", date: "2026-03-31" })), - exec([{ transaction_date: "2026-03-31", amount: "5.00", transaction_type: "fee" }]) - ); - expect(r.ledgerDuplicates).toBe(1); - expect(r.toInsert).toBe(0); - }); - - it("drops anything a statement already covers, whatever the amount", async () => { - // The primary guard. Amount-matching cannot see this case: the two sources - // decompose the same event differently — Frollo bundles the Wise fee into - // the transfer (10001.13) where the statement itemises it (10000.00 + 1.13) - // — so 38 Wise USD rows survived the amount guard while being the same - // money. A statement's billing_end_date is a hard watermark instead. - const r = await ingestFrolloCsv( - csv( - line({ id: "1", desc: "Interactive Brokers LLC", amount: "-10001.13", date: "2026-07-25" }), - line({ id: "2", desc: "HDR Global Services", amount: "10782.00", date: "2026-08-12" }) - ), - exec([], [{ last4: "1910", covered_to: "2026-07-26" }]) - ); - expect(r.coveredByStatement).toBe(1); - expect(r.toInsert).toBe(1); // only the post-watermark row survives - expect(r.statementWatermarks).toEqual([{ last4: "1910", coveredTo: "2026-07-26" }]); - }); - - it("keeps everything for an account that has no statement at all", async () => { - const r = await ingestFrolloCsv( - csv(line({ id: "1", amount: "-42.50", date: "2020-01-01" })), - exec([], [{ last4: "9999", covered_to: "2026-07-26" }]) - ); - expect(r.coveredByStatement).toBe(0); - expect(r.toInsert).toBe(1); - }); - - it("treats the watermark date itself as covered", async () => { - const r = await ingestFrolloCsv( - csv(line({ id: "1", amount: "-42.50", date: "2026-07-26" })), - exec([], [{ last4: "1910", covered_to: "2026-07-26" }]) - ); - expect(r.coveredByStatement).toBe(1); - expect(r.toInsert).toBe(0); - }); - - it("keeps everything when the ledger is empty", async () => { - const r = await ingestFrolloCsv( - csv( - line({ id: "1", desc: "A", amount: "-5.00", date: "2026-03-10" }), - line({ id: "2", desc: "B", amount: "-6.00", date: "2026-03-11" }) - ), - exec([]) - ); - expect(r.ledgerDuplicates).toBe(0); - expect(r.toInsert).toBe(2); - }); - - it("reports duplicates rather than silently discarding them", async () => { - // The owner's standing requirement: a row that disappears must be counted, - // because a missing transaction can mean several things and one of them is - // worth a follow-up. - const r = await ingestFrolloCsv( - csv(line({ id: "1", amount: "-42.50", date: "2026-03-10" })), - exec([{ transaction_date: "2026-03-10", amount: "42.50" }]) - ); - expect(r).toHaveProperty("ledgerDuplicates"); - expect(r.inScope).toBe(1); - expect(r.ledgerDuplicates + r.toInsert).toBe(r.inScope); - }); -}); diff --git a/src/app/api/frollo/ingest/route.ts b/src/app/api/frollo/ingest/route.ts deleted file mode 100644 index 535b577..0000000 --- a/src/app/api/frollo/ingest/route.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { NextRequest, NextResponse } from "next/server"; -import { queryRaw } from "@/lib/db"; -import { ingestFrolloCsv } from "@/lib/frollo-ingest"; - -/** - * Machine ingest endpoint for a Frollo transaction export. - * - * Sibling to /api/orders/ingest and /api/receipts/ingest, shaped like them. Auth - * is a shared secret rather than the Traefik `x-forwarded-user` header, because - * n8n calls this app-to-app with no browser session to forward. - * - * Its own token so it can be rotated without touching the order or receipt - * flows, each of which runs on a schedule nobody is watching. - * - * The body is the CSV itself, as text. The caller does not parse anything — - * parsing, scoping, de-duplication and the insert all live in one place - * (`frollo-ingest.ts`), shared with `scripts/import-frollo.mts`. - */ -function authorised(req: NextRequest): boolean { - const expected = process.env.FROLLO_INGEST_TOKEN; - if (!expected) return false; // fail closed when unconfigured - const got = req.headers.get("x-ingest-token"); - return !!got && got === expected; -} - -export async function POST(req: NextRequest) { - if (!authorised(req)) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); - - let body: { csv?: string; dryRun?: boolean; force?: boolean }; - try { - body = await req.json(); - } catch { - return NextResponse.json({ error: "invalid JSON" }, { status: 400 }); - } - if (!body?.csv || typeof body.csv !== "string") { - return NextResponse.json({ error: "csv (string) is required" }, { status: 400 }); - } - - try { - const report = await ingestFrolloCsv(body.csv, queryRaw, { - apply: !body.dryRun, - force: body.force === true, - }); - - // An automatic caller must not be able to write through an anomaly by - // accident: the run reports 409 and inserts nothing, so a human decides. - // Today's session is the argument — the de-duplication rule passed 29 tests - // and a clean dry run while still doubling three salary payments. - if (!report.applied && report.anomalies.length > 0 && !body.dryRun) { - return NextResponse.json({ kind: "needs_review", ...report }, { status: 409 }); - } - return NextResponse.json({ kind: "frollo", ...report }); - } catch (e) { - // A missing or renamed column throws rather than importing blank rows, and - // that is worth surfacing as loudly as an unparseable receipt. - const message = e instanceof Error ? e.message : String(e); - if (/missing expected column|empty CSV/i.test(message)) { - return NextResponse.json({ kind: "rejected", reason: message }, { status: 422 }); - } - return NextResponse.json({ error: message }, { status: 500 }); - } -} diff --git a/src/app/api/import/csv/route.ts b/src/app/api/import/csv/route.ts index d5942c9..68c6329 100644 --- a/src/app/api/import/csv/route.ts +++ b/src/app/api/import/csv/route.ts @@ -8,6 +8,12 @@ export async function POST(req: NextRequest) { const body = await req.json() as { bank_name: string; + /** + * Optional provenance label, stored on every row as `source`. With a + * `source_ref` per row it makes a re-import a no-op; without it this path + * has no idempotency at all. + */ + source?: string; transactions: { date: string; description: string; @@ -17,6 +23,8 @@ export async function POST(req: NextRequest) { foreign_currency_amount?: number; foreign_currency_code?: string; category?: string; + account?: string; + source_ref?: string; }[]; }; @@ -25,7 +33,7 @@ export async function POST(req: NextRequest) { } const tagId = await ensureTag("csv-import", "#8b5cf6"); - const inserted = await batchInsertCSVTransactions(user.id, body.transactions, tagId); + const inserted = await batchInsertCSVTransactions(user.id, body.transactions, tagId, body.source); return NextResponse.json({ inserted }, { status: 201 }); } diff --git a/src/app/api/import/statement-coverage/route.ts b/src/app/api/import/statement-coverage/route.ts new file mode 100644 index 0000000..2b0fc61 --- /dev/null +++ b/src/app/api/import/statement-coverage/route.ts @@ -0,0 +1,17 @@ +import { NextRequest, NextResponse } from "next/server"; +import { getCurrentUser } from "@/lib/auth"; +import { getStatementCoverage } from "@/lib/queries"; + +/** + * How far each account's statements already reach. + * + * Read by the CSV import modal so it can drop rows the ledger already holds. + * The first Frollo import went in without this and wrote 422 duplicates out of + * 550 — the same payments the statements already carried, itemised and + * converted. On the 2026-08-13 export the rule takes 2,607 rows down to 171. + */ +export async function GET(req: NextRequest) { + const user = await getCurrentUser(req); + if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 403 }); + return NextResponse.json({ coverage: await getStatementCoverage() }); +} diff --git a/src/components/csv-import-modal.tsx b/src/components/csv-import-modal.tsx index 8916241..08d5a34 100644 --- a/src/components/csv-import-modal.tsx +++ b/src/components/csv-import-modal.tsx @@ -2,10 +2,10 @@ import { useState, useRef, useEffect } from "react"; import { CATEGORIES, formatCategory } from "@/lib/categories"; -import { useImportCSV } from "@/lib/hooks"; +import { useImportCSV, useStatementCoverage } from "@/lib/hooks"; import { parseCSVRows, detectHasHeaders, getColumnLabels, getDataRows, applyMapping, - saveBankPreset, loadBankPresets, + saveBankPreset, loadBankPresets, splitByCoverage, inFileDuplicates, type DateFormat, type ColumnMapping, type ParsedTransaction, type BankPreset, } from "@/lib/csv-parser"; @@ -14,6 +14,8 @@ const TX_TYPES = ["debit", "credit", "payment", "refund", "fee", "interest", "tr type Step = "upload" | "map" | "review" | "done"; + + function ColSelect({ label, value, onChange, options, required, }: { @@ -54,6 +56,9 @@ export function CsvImportModal({ onClose }: { onClose: () => void }) { const [error, setError] = useState(""); const [presets, setPresets] = useState([]); const [insertedCount, setInsertedCount] = useState(0); + const [coveredRows, setCoveredRows] = useState([]); + const [showCovered, setShowCovered] = useState(false); + const coverage = useStatementCoverage(); useEffect(() => { setPresets(loadBankPresets()); }, []); @@ -111,7 +116,9 @@ export function CsvImportModal({ onClose }: { onClose: () => void }) { if (savePreset) { saveBankPreset({ bankName: bankName.trim(), mapping, dateFormat }); } - setEditedRows(parsed); + const { keep, covered } = splitByCoverage(parsed, coverage.data ?? []); + setEditedRows(keep); + setCoveredRows(covered); setStep("review"); } @@ -120,7 +127,13 @@ export function CsvImportModal({ onClose }: { onClose: () => void }) { const valid = editedRows.filter((r) => r.date && r.amount > 0 && r.description); if (!valid.length) { setError("No valid rows to import"); return; } try { - const result = await importCSV.mutateAsync({ bank_name: bankName, transactions: valid }); + const result = await importCSV.mutateAsync({ + bank_name: bankName, + // Provenance, so a second import of the same file inserts nothing — + // paired with each row's source_ref where the file carries an id. + source: bankName.trim().toLowerCase().replace(/\s+/g, "-"), + transactions: valid, + }); setInsertedCount(result.inserted); setStep("done"); } catch (e) { @@ -286,7 +299,14 @@ export function CsvImportModal({ onClose }: { onClose: () => void }) {
setMapping((m) => ({ ...m, merchantCol: v || undefined }))} options={columnLabels} /> setMapping((m) => ({ ...m, categoryCol: v || undefined }))} options={columnLabels} /> + setMapping((m) => ({ ...m, accountCol: v || undefined }))} options={columnLabels} /> + setMapping((m) => ({ ...m, sourceRefCol: v || undefined }))} options={columnLabels} />
+

+ Map Account when the file covers more than one account: rows already + covered by that account's statements are then left out. Map Row ID to + make re-importing the same file do nothing. +