frollo: don't import what the statements already cover

"We should not be importing from Frollo what we already have from statements"
(owner). The amount+direction guard could not deliver that, because the two
sources decompose the same event differently: Frollo bundles the Wise fee into
the transfer (10001.13) where the statement itemises it (10000.00 + 1.13). So 38
Wise USD rows passed the amount guard as new while being the same money, and were
the reason the transactions view showed a USD figure where every neighbouring row
showed AUD.

A statement's billing_end_date is a hard watermark: everything on that account up
to that date is already in the ledger, itemised and converted. Guard 1 now drops
any feed row at or before its account's newest statement. Guard 2 (amount +
direction) stays as the net for accounts that have no statement at all.

Note this is the coverage test the first import needed and got wrong. That one
asked whether a row's date fell inside a statement's min-max window, which for
periods spanning 182 to 460 days swallows a year and answers nothing. The
watermark asks a question that has an answer: up to what date is this account
complete?

Matching is on last4, verified against the live statement set — the eight
in-scope accounts with statements each map to one bank, no cross-bank collision.
The watermarks are printed by the CLI so a wrong boundary is visible rather than
inferred from a row count.

Re-imported from empty: 425 covered by statement, 15 amount twins, 110 inserted.
Exactly one row now carries a foreign currency with no AUD figure — the
2026-08-12 HDR salary, which is the genuinely-new pre-statement row this feed
exists for. Was 39.

scripts/import-frollo.mts

@@ -99,10 +99,17 @@ console.log();
 
 console.log("LEDGER");
 console.log(`  already imported : ${String(report.alreadyImported).padStart(5)}`);
-// Printed even when zero. This is the count that was missing on 2026-08-13,
+// Printed even when zero. These are the counts that were missing on
-// when the whole file was written on top of a ledger that already held 77% of
+// 2026-08-13, when the whole file was written on top of a ledger that already
-// it; a number you have to go looking for is a number nobody looks at.
+// held 77% of it; a number you have to go looking for is a number nobody looks
-console.log(`  already on ledger: ${String(report.ledgerDuplicates).padStart(5)}  (statement twin within ${LEDGER_MATCH_DAYS} days)`);
+// at. The watermarks are printed too, so a wrong coverage boundary is visible
+// rather than inferred from a row count.
+console.log(`  covered by stmt  : ${String(report.coveredByStatement).padStart(5)}  (dated on or before the account's newest statement)`);
+console.log(`  amount twin      : ${String(report.ledgerDuplicates).padStart(5)}  (same amount + direction within ${LEDGER_MATCH_DAYS} days)`);
+if (report.statementWatermarks.length > 0) {
+  console.log("  statements cover:");
+  for (const w of report.statementWatermarks) console.log(`    ${w.last4}  up to ${w.coveredTo}`);
+}
 console.log(`  new to insert    : ${String(report.toInsert).padStart(5)}`);
 
 if (report.anomalies.length > 0) {

src/__tests__/unit/frollo-ingest.test.ts

@@ -46,9 +46,13 @@ const csv = (...lines: string[]) => [HEADER, ...lines].join("\n");
  * which is crude but keeps the test free of a live connection — and the guard
  * is pure logic over what comes back.
  */
-function exec(prior: { transaction_date: string; amount: string; transaction_type?: string }[]): SqlExecutor {
+function exec(
+  prior: { transaction_date: string; amount: string; transaction_type?: string }[],
+  coverage: { last4: string; covered_to: string }[] = []
+): SqlExecutor {
   return (async (sql: string) => {
     if (sql.includes("WHERE source = $1")) return [];
+    if (sql.includes("FROM statements")) return coverage;
     if (sql.includes("superseded_by_id IS NULL")) return prior.map((p) => ({ transaction_type: "debit", ...p }));
     return [];
   }) as SqlExecutor;
@@ -162,6 +166,42 @@ describe("ledger-duplicate guard", () => {
     expect(r.toInsert).toBe(0);
   });
 
+  it("drops anything a statement already covers, whatever the amount", async () => {
+    // The primary guard. Amount-matching cannot see this case: the two sources
+    // decompose the same event differently — Frollo bundles the Wise fee into
+    // the transfer (10001.13) where the statement itemises it (10000.00 + 1.13)
+    // — so 38 Wise USD rows survived the amount guard while being the same
+    // money. A statement's billing_end_date is a hard watermark instead.
+    const r = await ingestFrolloCsv(
+      csv(
+        line({ id: "1", desc: "Interactive Brokers LLC", amount: "-10001.13", date: "2026-07-25" }),
+        line({ id: "2", desc: "HDR Global Services", amount: "10782.00", date: "2026-08-12" })
+      ),
+      exec([], [{ last4: "1910", covered_to: "2026-07-26" }])
+    );
+    expect(r.coveredByStatement).toBe(1);
+    expect(r.toInsert).toBe(1); // only the post-watermark row survives
+    expect(r.statementWatermarks).toEqual([{ last4: "1910", coveredTo: "2026-07-26" }]);
+  });
+
+  it("keeps everything for an account that has no statement at all", async () => {
+    const r = await ingestFrolloCsv(
+      csv(line({ id: "1", amount: "-42.50", date: "2020-01-01" })),
+      exec([], [{ last4: "9999", covered_to: "2026-07-26" }])
+    );
+    expect(r.coveredByStatement).toBe(0);
+    expect(r.toInsert).toBe(1);
+  });
+
+  it("treats the watermark date itself as covered", async () => {
+    const r = await ingestFrolloCsv(
+      csv(line({ id: "1", amount: "-42.50", date: "2026-07-26" })),
+      exec([], [{ last4: "1910", covered_to: "2026-07-26" }])
+    );
+    expect(r.coveredByStatement).toBe(1);
+    expect(r.toInsert).toBe(0);
+  });
+
   it("keeps everything when the ledger is empty", async () => {
     const r = await ingestFrolloCsv(
       csv(

src/lib/frollo-ingest.ts

@@ -1,6 +1,7 @@
 import {
   accountReport,
   dedupe,
+  last4,
   readRows,
   skipReason,
   toLedgerRow,
@@ -53,16 +54,6 @@ export const LARGE_BATCH = 200;
  */
 export const LEDGER_MATCH_DAYS = 3;
 
-/**
- * A date column, as midnight UTC, from whatever the driver handed back.
- *
- * `pg` maps a Postgres DATE to a JS `Date` in local time, while Prisma and the
- * CSV both yield `"YYYY-MM-DD"` strings. Reading one as the other is silent:
- * `String(new Date()).slice(0, 10)` is `"Wed Mar 10"`, which parses to NaN, and
- * a guard built on it skips every row it was meant to compare — the first dry
- * run of this code reported 550 rows to insert and zero duplicates against a
- * ledger holding 422 of them.
- */
 /**
  * Ledger transaction types that mean money arriving.
  *
@@ -75,6 +66,16 @@ export const LEDGER_MATCH_DAYS = 3;
  */
 const INFLOW_TYPES = new Set(["credit", "refund"]);
 
+/**
+ * A date column, as midnight UTC, from whatever the driver handed back.
+ *
+ * `pg` maps a Postgres DATE to a JS `Date` in local time, while Prisma and the
+ * CSV both yield `"YYYY-MM-DD"` strings. Reading one as the other is silent:
+ * `String(new Date()).slice(0, 10)` is `"Wed Mar 10"`, which parses to NaN, and
+ * a guard built on it skips every row it was meant to compare — the first dry
+ * run of this code reported 550 rows to insert and zero duplicates against a
+ * ledger holding 422 of them.
+ */
 function dayMs(v: unknown): number {
   if (v instanceof Date) return Date.UTC(v.getFullYear(), v.getMonth(), v.getDate());
   const m = /^(\d{4})-(\d{2})-(\d{2})/.exec(String(v));
@@ -101,9 +102,18 @@ export interface IngestReport {
   suspectRepeats: { description: string; transactionDate: string; amount: number; idGap: number }[];
   alreadyImported: number;
   /**
-   * Rows dropped because the ledger already holds the same event from a
+   * Rows dropped because a statement already covers that account up to that
-   * statement. Counted and reported, never silent: a Frollo row vanishing can
+   * date. The primary guard — see `statementWatermarks`.
-   * mean several things and one of them is worth following up.
+   */
+  coveredByStatement: number;
+  /** Per-account coverage boundary used above, for the report. */
+  statementWatermarks: { last4: string; coveredTo: string }[];
+  /**
+   * Rows dropped because the ledger holds the same amount, same direction,
+   * within a few days. The secondary guard, for accounts with no statement.
+   *
+   * Both counts are reported, never silent: a Frollo row vanishing can mean
+   * several things and one of them is worth following up.
    */
   ledgerDuplicates: number;
   toInsert: number;
@@ -195,7 +205,59 @@ export async function ingestFrolloCsv(
   const seen = new Set(existing.map((r) => r.source_ref));
   const notYetImported = ledger.filter((r) => !seen.has(r.sourceRef));
 
-  // Drop rows the ledger already holds from a statement.
+  // GUARD 1 — statement coverage. Do not import what the statements already
+  // have.
+  //
+  // A statement's `billing_end_date` is a hard watermark: everything on that
+  // account up to that date is already in the ledger, itemised and converted.
+  // Amount-matching cannot replace this, because the two sources decompose the
+  // same event differently — Frollo bundles the Wise fee into the transfer
+  // (10001.13) where the statement itemises it (10000.00 + 1.13), so 38 Wise USD
+  // rows survived the amount guard as "new" while being the same money.
+  //
+  // This is the coverage test the first import needed and got wrong. It asked
+  // whether a row's date fell inside a statement's min–max *window*, which for
+  // periods spanning 182 to 460 days swallows a year and answers nothing. The
+  // watermark asks a different and answerable question: up to what date is this
+  // account complete?
+  //
+  // Matching is on last4 alone. Verified against the live statement set: the
+  // eight in-scope accounts with statements each map to exactly one bank, no
+  // cross-bank collision. The per-account watermarks are reported so a wrong one
+  // is visible rather than inferred from a row count.
+  const coverage = await exec<{ last4: string; covered_to: unknown }>(
+    `SELECT right(regexp_replace(account_number, '[^0-9]', '', 'g'), 4) AS last4,
+            MAX(billing_end_date) AS covered_to
+       FROM statements
+      WHERE account_number IS NOT NULL
+      GROUP BY 1`,
+    []
+  );
+  const watermark = new Map<string, number>();
+  for (const c of coverage) {
+    if (!c.last4) continue; // Wise files one statement as account_number 'N/A'
+    const t = dayMs(c.covered_to);
+    if (!Number.isFinite(t)) continue;
+    watermark.set(c.last4, Math.max(watermark.get(c.last4) ?? -Infinity, t));
+  }
+
+  let coveredByStatement = 0;
+  const uncovered = notYetImported.filter((r) => {
+    const w = watermark.get(last4(r.sourceAccount));
+    if (w === undefined) return true; // no statement for this account, ever
+    if (dayMs(r.transactionDate) <= w) {
+      coveredByStatement += 1;
+      return false;
+    }
+    return true;
+  });
+
+  const statementWatermarks = [...watermark.entries()]
+    .filter(([l4]) => notYetImported.some((r) => last4(r.sourceAccount) === l4))
+    .map(([l4, t]) => ({ last4: l4, coveredTo: new Date(t).toISOString().slice(0, 10) }))
+    .sort((a, b) => a.last4.localeCompare(b.last4));
+
+  // GUARD 2 — drop rows the ledger already holds from a statement.
   //
   // This guard is the whole lesson of the 2026-08-13 first import. The feed was
   // scoped to "accounts that issue no monthly statement", but that was asserted,
@@ -209,9 +271,9 @@ export async function ingestFrolloCsv(
   // check that was run instead compared each row's date against the statement's
   // min-max window, which for accounts whose statements span 182 to 460 days
   // swallows a year and cannot distinguish covered from uncovered at all.
-  const dates = notYetImported.map((r) => dayMs(r.transactionDate)).filter(Number.isFinite);
+  const dates = uncovered.map((r) => dayMs(r.transactionDate)).filter(Number.isFinite);
   let ledgerDuplicates = 0;
-  let fresh = notYetImported;
+  let fresh = uncovered;
   if (dates.length > 0) {
     const pad = LEDGER_MATCH_DAYS * 86_400_000;
     const from = new Date(Math.min(...dates) - pad).toISOString().slice(0, 10);
@@ -252,7 +314,7 @@ export async function ingestFrolloCsv(
       else byAmount.set(k, [t]);
     }
 
-    fresh = notYetImported.filter((r) => {
+    fresh = uncovered.filter((r) => {
       const candidates = byAmount.get(key(r.amount, r.transactionType === "credit"));
       if (!candidates) return true;
       const t = dayMs(r.transactionDate);
@@ -277,6 +339,8 @@ export async function ingestFrolloCsv(
     duplicatesDropped: dropped.length,
     suspectRepeats,
     alreadyImported: ledger.length - notYetImported.length,
+    coveredByStatement,
+    statementWatermarks,
     ledgerDuplicates,
     toInsert: fresh.length,
   };