From b82c4570bd281ae750b7e2d9a70cb429dfc7a515 Mon Sep 17 00:00:00 2001 From: siddharthd Date: Thu, 13 Aug 2026 12:36:08 +1000 Subject: [PATCH] frollo: don't import what the statements already cover MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "We should not be importing from Frollo what we already have from statements" (owner). The amount+direction guard could not deliver that, because the two sources decompose the same event differently: Frollo bundles the Wise fee into the transfer (10001.13) where the statement itemises it (10000.00 + 1.13). So 38 Wise USD rows passed the amount guard as new while being the same money, and were the reason the transactions view showed a USD figure where every neighbouring row showed AUD. A statement's billing_end_date is a hard watermark: everything on that account up to that date is already in the ledger, itemised and converted. Guard 1 now drops any feed row at or before its account's newest statement. Guard 2 (amount + direction) stays as the net for accounts that have no statement at all. Note this is the coverage test the first import needed and got wrong. That one asked whether a row's date fell inside a statement's min-max window, which for periods spanning 182 to 460 days swallows a year and answers nothing. The watermark asks a question that has an answer: up to what date is this account complete? Matching is on last4, verified against the live statement set — the eight in-scope accounts with statements each map to one bank, no cross-bank collision. The watermarks are printed by the CLI so a wrong boundary is visible rather than inferred from a row count. Re-imported from empty: 425 covered by statement, 15 amount twins, 110 inserted. Exactly one row now carries a foreign currency with no AUD figure — the 2026-08-12 HDR salary, which is the genuinely-new pre-statement row this feed exists for. Was 39. --- scripts/import-frollo.mts | 15 +++- src/__tests__/unit/frollo-ingest.test.ts | 42 +++++++++- src/lib/frollo-ingest.ts | 98 ++++++++++++++++++++---- 3 files changed, 133 insertions(+), 22 deletions(-) diff --git a/scripts/import-frollo.mts b/scripts/import-frollo.mts index ff05fdd..a0693bb 100644 --- a/scripts/import-frollo.mts +++ b/scripts/import-frollo.mts @@ -99,10 +99,17 @@ console.log(); console.log("LEDGER"); console.log(` already imported : ${String(report.alreadyImported).padStart(5)}`); -// Printed even when zero. This is the count that was missing on 2026-08-13, -// when the whole file was written on top of a ledger that already held 77% of -// it; a number you have to go looking for is a number nobody looks at. -console.log(` already on ledger: ${String(report.ledgerDuplicates).padStart(5)} (statement twin within ${LEDGER_MATCH_DAYS} days)`); +// Printed even when zero. These are the counts that were missing on +// 2026-08-13, when the whole file was written on top of a ledger that already +// held 77% of it; a number you have to go looking for is a number nobody looks +// at. The watermarks are printed too, so a wrong coverage boundary is visible +// rather than inferred from a row count. +console.log(` covered by stmt : ${String(report.coveredByStatement).padStart(5)} (dated on or before the account's newest statement)`); +console.log(` amount twin : ${String(report.ledgerDuplicates).padStart(5)} (same amount + direction within ${LEDGER_MATCH_DAYS} days)`); +if (report.statementWatermarks.length > 0) { + console.log(" statements cover:"); + for (const w of report.statementWatermarks) console.log(` ${w.last4} up to ${w.coveredTo}`); +} console.log(` new to insert : ${String(report.toInsert).padStart(5)}`); if (report.anomalies.length > 0) { diff --git a/src/__tests__/unit/frollo-ingest.test.ts b/src/__tests__/unit/frollo-ingest.test.ts index a551409..32ef25b 100644 --- a/src/__tests__/unit/frollo-ingest.test.ts +++ b/src/__tests__/unit/frollo-ingest.test.ts @@ -46,9 +46,13 @@ const csv = (...lines: string[]) => [HEADER, ...lines].join("\n"); * which is crude but keeps the test free of a live connection — and the guard * is pure logic over what comes back. */ -function exec(prior: { transaction_date: string; amount: string; transaction_type?: string }[]): SqlExecutor { +function exec( + prior: { transaction_date: string; amount: string; transaction_type?: string }[], + coverage: { last4: string; covered_to: string }[] = [] +): SqlExecutor { return (async (sql: string) => { if (sql.includes("WHERE source = $1")) return []; + if (sql.includes("FROM statements")) return coverage; if (sql.includes("superseded_by_id IS NULL")) return prior.map((p) => ({ transaction_type: "debit", ...p })); return []; }) as SqlExecutor; @@ -162,6 +166,42 @@ describe("ledger-duplicate guard", () => { expect(r.toInsert).toBe(0); }); + it("drops anything a statement already covers, whatever the amount", async () => { + // The primary guard. Amount-matching cannot see this case: the two sources + // decompose the same event differently — Frollo bundles the Wise fee into + // the transfer (10001.13) where the statement itemises it (10000.00 + 1.13) + // — so 38 Wise USD rows survived the amount guard while being the same + // money. A statement's billing_end_date is a hard watermark instead. + const r = await ingestFrolloCsv( + csv( + line({ id: "1", desc: "Interactive Brokers LLC", amount: "-10001.13", date: "2026-07-25" }), + line({ id: "2", desc: "HDR Global Services", amount: "10782.00", date: "2026-08-12" }) + ), + exec([], [{ last4: "1910", covered_to: "2026-07-26" }]) + ); + expect(r.coveredByStatement).toBe(1); + expect(r.toInsert).toBe(1); // only the post-watermark row survives + expect(r.statementWatermarks).toEqual([{ last4: "1910", coveredTo: "2026-07-26" }]); + }); + + it("keeps everything for an account that has no statement at all", async () => { + const r = await ingestFrolloCsv( + csv(line({ id: "1", amount: "-42.50", date: "2020-01-01" })), + exec([], [{ last4: "9999", covered_to: "2026-07-26" }]) + ); + expect(r.coveredByStatement).toBe(0); + expect(r.toInsert).toBe(1); + }); + + it("treats the watermark date itself as covered", async () => { + const r = await ingestFrolloCsv( + csv(line({ id: "1", amount: "-42.50", date: "2026-07-26" })), + exec([], [{ last4: "1910", covered_to: "2026-07-26" }]) + ); + expect(r.coveredByStatement).toBe(1); + expect(r.toInsert).toBe(0); + }); + it("keeps everything when the ledger is empty", async () => { const r = await ingestFrolloCsv( csv( diff --git a/src/lib/frollo-ingest.ts b/src/lib/frollo-ingest.ts index e8dff85..dc7411f 100644 --- a/src/lib/frollo-ingest.ts +++ b/src/lib/frollo-ingest.ts @@ -1,6 +1,7 @@ import { accountReport, dedupe, + last4, readRows, skipReason, toLedgerRow, @@ -53,16 +54,6 @@ export const LARGE_BATCH = 200; */ export const LEDGER_MATCH_DAYS = 3; -/** - * A date column, as midnight UTC, from whatever the driver handed back. - * - * `pg` maps a Postgres DATE to a JS `Date` in local time, while Prisma and the - * CSV both yield `"YYYY-MM-DD"` strings. Reading one as the other is silent: - * `String(new Date()).slice(0, 10)` is `"Wed Mar 10"`, which parses to NaN, and - * a guard built on it skips every row it was meant to compare — the first dry - * run of this code reported 550 rows to insert and zero duplicates against a - * ledger holding 422 of them. - */ /** * Ledger transaction types that mean money arriving. * @@ -75,6 +66,16 @@ export const LEDGER_MATCH_DAYS = 3; */ const INFLOW_TYPES = new Set(["credit", "refund"]); +/** + * A date column, as midnight UTC, from whatever the driver handed back. + * + * `pg` maps a Postgres DATE to a JS `Date` in local time, while Prisma and the + * CSV both yield `"YYYY-MM-DD"` strings. Reading one as the other is silent: + * `String(new Date()).slice(0, 10)` is `"Wed Mar 10"`, which parses to NaN, and + * a guard built on it skips every row it was meant to compare — the first dry + * run of this code reported 550 rows to insert and zero duplicates against a + * ledger holding 422 of them. + */ function dayMs(v: unknown): number { if (v instanceof Date) return Date.UTC(v.getFullYear(), v.getMonth(), v.getDate()); const m = /^(\d{4})-(\d{2})-(\d{2})/.exec(String(v)); @@ -101,9 +102,18 @@ export interface IngestReport { suspectRepeats: { description: string; transactionDate: string; amount: number; idGap: number }[]; alreadyImported: number; /** - * Rows dropped because the ledger already holds the same event from a - * statement. Counted and reported, never silent: a Frollo row vanishing can - * mean several things and one of them is worth following up. + * Rows dropped because a statement already covers that account up to that + * date. The primary guard — see `statementWatermarks`. + */ + coveredByStatement: number; + /** Per-account coverage boundary used above, for the report. */ + statementWatermarks: { last4: string; coveredTo: string }[]; + /** + * Rows dropped because the ledger holds the same amount, same direction, + * within a few days. The secondary guard, for accounts with no statement. + * + * Both counts are reported, never silent: a Frollo row vanishing can mean + * several things and one of them is worth following up. */ ledgerDuplicates: number; toInsert: number; @@ -195,7 +205,59 @@ export async function ingestFrolloCsv( const seen = new Set(existing.map((r) => r.source_ref)); const notYetImported = ledger.filter((r) => !seen.has(r.sourceRef)); - // Drop rows the ledger already holds from a statement. + // GUARD 1 — statement coverage. Do not import what the statements already + // have. + // + // A statement's `billing_end_date` is a hard watermark: everything on that + // account up to that date is already in the ledger, itemised and converted. + // Amount-matching cannot replace this, because the two sources decompose the + // same event differently — Frollo bundles the Wise fee into the transfer + // (10001.13) where the statement itemises it (10000.00 + 1.13), so 38 Wise USD + // rows survived the amount guard as "new" while being the same money. + // + // This is the coverage test the first import needed and got wrong. It asked + // whether a row's date fell inside a statement's min–max *window*, which for + // periods spanning 182 to 460 days swallows a year and answers nothing. The + // watermark asks a different and answerable question: up to what date is this + // account complete? + // + // Matching is on last4 alone. Verified against the live statement set: the + // eight in-scope accounts with statements each map to exactly one bank, no + // cross-bank collision. The per-account watermarks are reported so a wrong one + // is visible rather than inferred from a row count. + const coverage = await exec<{ last4: string; covered_to: unknown }>( + `SELECT right(regexp_replace(account_number, '[^0-9]', '', 'g'), 4) AS last4, + MAX(billing_end_date) AS covered_to + FROM statements + WHERE account_number IS NOT NULL + GROUP BY 1`, + [] + ); + const watermark = new Map(); + for (const c of coverage) { + if (!c.last4) continue; // Wise files one statement as account_number 'N/A' + const t = dayMs(c.covered_to); + if (!Number.isFinite(t)) continue; + watermark.set(c.last4, Math.max(watermark.get(c.last4) ?? -Infinity, t)); + } + + let coveredByStatement = 0; + const uncovered = notYetImported.filter((r) => { + const w = watermark.get(last4(r.sourceAccount)); + if (w === undefined) return true; // no statement for this account, ever + if (dayMs(r.transactionDate) <= w) { + coveredByStatement += 1; + return false; + } + return true; + }); + + const statementWatermarks = [...watermark.entries()] + .filter(([l4]) => notYetImported.some((r) => last4(r.sourceAccount) === l4)) + .map(([l4, t]) => ({ last4: l4, coveredTo: new Date(t).toISOString().slice(0, 10) })) + .sort((a, b) => a.last4.localeCompare(b.last4)); + + // GUARD 2 — drop rows the ledger already holds from a statement. // // This guard is the whole lesson of the 2026-08-13 first import. The feed was // scoped to "accounts that issue no monthly statement", but that was asserted, @@ -209,9 +271,9 @@ export async function ingestFrolloCsv( // check that was run instead compared each row's date against the statement's // min-max window, which for accounts whose statements span 182 to 460 days // swallows a year and cannot distinguish covered from uncovered at all. - const dates = notYetImported.map((r) => dayMs(r.transactionDate)).filter(Number.isFinite); + const dates = uncovered.map((r) => dayMs(r.transactionDate)).filter(Number.isFinite); let ledgerDuplicates = 0; - let fresh = notYetImported; + let fresh = uncovered; if (dates.length > 0) { const pad = LEDGER_MATCH_DAYS * 86_400_000; const from = new Date(Math.min(...dates) - pad).toISOString().slice(0, 10); @@ -252,7 +314,7 @@ export async function ingestFrolloCsv( else byAmount.set(k, [t]); } - fresh = notYetImported.filter((r) => { + fresh = uncovered.filter((r) => { const candidates = byAmount.get(key(r.amount, r.transactionType === "credit")); if (!candidates) return true; const t = dayMs(r.transactionDate); @@ -277,6 +339,8 @@ export async function ingestFrolloCsv( duplicatesDropped: dropped.length, suspectRepeats, alreadyImported: ledger.length - notYetImported.length, + coveredByStatement, + statementWatermarks, ledgerDuplicates, toInsert: fresh.length, };