csv import: exclude what the statements already cover, and delete the Frollo importer

"This is becoming too complex... Frollo should be done through that [the manual
CSV import]" (owner). It was right: a bespoke importer, an API route, a CLI, two
scheduled n8n workflows and a shared secret existed to do what the CSV import
modal already did, minus one rule.

That rule is statement coverage, and it turns out to be the whole thing. Applying
each account's newest billing_end_date as a watermark takes the real 2,607-row
Frollo export down to 171 rows — with no account allowlist, no credit-card
exclusion and no Frollo-specific scoping at all. Cards drop out on their own
because their statements are current; the 46 card rows that survive are genuinely
post-statement. Every bit of the bespoke apparatus was doing by hand what one
query does generically.

Deleted: src/lib/frollo-csv.ts, src/lib/frollo-ingest.ts, scripts/import-frollo.mts,
src/app/api/frollo/, both test files, the FROLLO_INGEST_TOKEN wiring, and the n8n
Frollo Import + Frollo Freshness Check workflows.

Added to the shared CSV path, so every import benefits:

  - getStatementCoverage() + /api/import/statement-coverage. The review step
    leaves out rows an account's statements already cover and says how many, with
    the rows one click away. Only applies when an account column is mapped and
    that account has statements — a row is never dropped on a guess.
  - Optional Account and Row ID columns in the mapper. Account drives the
    watermark and is stored as source_account; Row ID becomes source_ref.
  - An in-file duplicate warning. A CDR re-consent re-exports history under
    fresh ids, so source_ref cannot see it — 385 twins in one 2,563-row export
    doubled every salary payment, and that is not visible by eye in a review
    table.

Two pre-existing bugs in that path, both of which this plan depends on:

  - The category chosen in the review step was accepted by
    batchInsertCSVTransactions and then left out of the INSERT column list, so it
    was silently discarded and the trigger wrote 'other'. Not cosmetic: an
    uncategorised credit is admitted by NET_SPEND_ROWS and negated by
    SPEND_SIGNED, so 62 imported transfers cancelled $74,338 of spend while
    counting as no income.
  - The path had no idempotency whatsoever. row_index is assigned MAX+1 on every
    run, which makes uq_transaction_identity structurally unable to fire, so a
    second import of the same file duplicated all of it. Now writes source +
    source_ref with ON CONFLICT DO NOTHING.

awaitsStatementLine()'s removal note is kept but rewritten: it no longer points
at a deleted file, and the lesson stands — the queue jump from 8 to 558 was the
measurement, not the noise.

src/lib/csv-parser.ts

@@ -9,6 +9,22 @@ export interface ColumnMapping {
   creditCol?: string;
   merchantCol?: string;
   categoryCol?: string;
+  /**
+   * Column holding the account number, when the file covers more than one.
+   *
+   * Optional and usually absent — a bank's own export is one account per file.
+   * An aggregator export (Frollo) is not, and without this every row would be
+   * measured against the same statement coverage.
+   */
+  accountCol?: string;
+  /**
+   * Column holding the provider's own row id, when it has one.
+   *
+   * Stored as `source_ref` so re-importing the same file inserts nothing. The
+   * importer has no other idempotency: `row_index` is assigned MAX+1 on every
+   * run, which makes `uq_transaction_identity` structurally unable to fire.
+   */
+  sourceRefCol?: string;
 }
 
 export interface BankPreset {
@@ -24,6 +40,10 @@ export interface ParsedTransaction {
   transaction_type: string;
   merchant_name?: string;
   category?: string;
+  /** Raw account identifier from the file, when a column was mapped. */
+  account?: string;
+  /** Provider row id, when a column was mapped. */
+  source_ref?: string;
 }
 
 export function parseCSVRows(text: string): string[][] {
@@ -108,6 +128,8 @@ export function applyMapping(
   const descIdx = idx(mapping.descriptionCol);
   const merchantIdx = mapping.merchantCol ? idx(mapping.merchantCol) : -1;
   const categoryIdx = mapping.categoryCol ? idx(mapping.categoryCol) : -1;
+  const accountIdx = mapping.accountCol ? idx(mapping.accountCol) : -1;
+  const sourceRefIdx = mapping.sourceRefCol ? idx(mapping.sourceRefCol) : -1;
 
   const results: ParsedTransaction[] = [];
   for (const row of dataRows) {
@@ -138,6 +160,8 @@ export function applyMapping(
     const tx: ParsedTransaction = { date, description, amount, transaction_type };
     if (merchantIdx >= 0 && row[merchantIdx]) tx.merchant_name = row[merchantIdx].trim();
     if (categoryIdx >= 0 && row[categoryIdx]) tx.category = row[categoryIdx].trim();
+    if (accountIdx >= 0 && row[accountIdx]) tx.account = row[accountIdx].trim();
+    if (sourceRefIdx >= 0 && row[sourceRefIdx]) tx.source_ref = row[sourceRefIdx].trim();
     results.push(tx);
   }
   return results;
@@ -154,3 +178,57 @@ export function loadBankPresets(): BankPreset[] {
   try { return JSON.parse(localStorage.getItem("csv-presets") || "[]"); }
   catch { return []; }
 }
+
+/** Last four digits of an account identifier, however it is written. */
+export function last4(s: string): string {
+  const d = s.replace(/\D/g, "");
+  return d.length >= 4 ? d.slice(-4) : "";
+}
+
+/**
+ * Splits parsed rows into what to import and what the statements already have.
+ *
+ * A statement's end date is a hard watermark: everything on that account up to
+ * it is already in the ledger, itemised and converted. Without this the first
+ * aggregator import wrote 422 duplicates out of 550 rows; with it the same
+ * 2,607-row export offers 171.
+ *
+ * Rows are only excluded when an account column was mapped AND that account has
+ * statements. Anything else is imported — a row is never dropped on a guess.
+ */
+export function splitByCoverage(
+  rows: ParsedTransaction[],
+  coverage: { last4: string; coveredTo: string }[]
+): { keep: ParsedTransaction[]; covered: ParsedTransaction[] } {
+  const wm = new Map(coverage.map((c) => [c.last4, c.coveredTo]));
+  const keep: ParsedTransaction[] = [];
+  const covered: ParsedTransaction[] = [];
+  for (const r of rows) {
+    const to = r.account ? wm.get(last4(r.account)) : undefined;
+    // Both are YYYY-MM-DD, so a string compare is a date compare.
+    if (to && r.date <= to) covered.push(r);
+    else keep.push(r);
+  }
+  return { keep, covered };
+}
+
+/**
+ * Rows that duplicate another row in the SAME file.
+ *
+ * Distinct from the coverage check, and not catchable by `source_ref`: a CDR
+ * re-consent makes the provider re-export an account's whole history under
+ * fresh ids while the originals survive, so the twins are identical in
+ * everything except the id. That put 385 duplicates in one 2,563-row export and
+ * doubled every salary payment. 385 rows is not something you spot by eye in a
+ * review table, so it is counted here.
+ */
+export function inFileDuplicates(rows: ParsedTransaction[]): number {
+  const seen = new Set<string>();
+  let n = 0;
+  for (const r of rows) {
+    const key = [r.date, r.description, r.amount, r.transaction_type, r.account ?? ""].join("|");
+    if (seen.has(key)) n += 1;
+    else seen.add(key);
+  }
+  return n;
+}