csv import: exclude what the statements already cover, and delete the Frollo importer

"This is becoming too complex... Frollo should be done through that [the manual
CSV import]" (owner). It was right: a bespoke importer, an API route, a CLI, two
scheduled n8n workflows and a shared secret existed to do what the CSV import
modal already did, minus one rule.

That rule is statement coverage, and it turns out to be the whole thing. Applying
each account's newest billing_end_date as a watermark takes the real 2,607-row
Frollo export down to 171 rows — with no account allowlist, no credit-card
exclusion and no Frollo-specific scoping at all. Cards drop out on their own
because their statements are current; the 46 card rows that survive are genuinely
post-statement. Every bit of the bespoke apparatus was doing by hand what one
query does generically.

Deleted: src/lib/frollo-csv.ts, src/lib/frollo-ingest.ts, scripts/import-frollo.mts,
src/app/api/frollo/, both test files, the FROLLO_INGEST_TOKEN wiring, and the n8n
Frollo Import + Frollo Freshness Check workflows.

Added to the shared CSV path, so every import benefits:

  - getStatementCoverage() + /api/import/statement-coverage. The review step
    leaves out rows an account's statements already cover and says how many, with
    the rows one click away. Only applies when an account column is mapped and
    that account has statements — a row is never dropped on a guess.
  - Optional Account and Row ID columns in the mapper. Account drives the
    watermark and is stored as source_account; Row ID becomes source_ref.
  - An in-file duplicate warning. A CDR re-consent re-exports history under
    fresh ids, so source_ref cannot see it — 385 twins in one 2,563-row export
    doubled every salary payment, and that is not visible by eye in a review
    table.

Two pre-existing bugs in that path, both of which this plan depends on:

  - The category chosen in the review step was accepted by
    batchInsertCSVTransactions and then left out of the INSERT column list, so it
    was silently discarded and the trigger wrote 'other'. Not cosmetic: an
    uncategorised credit is admitted by NET_SPEND_ROWS and negated by
    SPEND_SIGNED, so 62 imported transfers cancelled $74,338 of spend while
    counting as no income.
  - The path had no idempotency whatsoever. row_index is assigned MAX+1 on every
    run, which makes uq_transaction_identity structurally unable to fire, so a
    second import of the same file duplicated all of it. Now writes source +
    source_ref with ON CONFLICT DO NOTHING.

awaitsStatementLine()'s removal note is kept but rewritten: it no longer points
at a deleted file, and the lesson stands — the queue jump from 8 to 558 was the
measurement, not the noise.

src/components/csv-import-modal.tsx

@@ -2,10 +2,10 @@
 
 import { useState, useRef, useEffect } from "react";
 import { CATEGORIES, formatCategory } from "@/lib/categories";
-import { useImportCSV } from "@/lib/hooks";
+import { useImportCSV, useStatementCoverage } from "@/lib/hooks";
 import {
   parseCSVRows, detectHasHeaders, getColumnLabels, getDataRows, applyMapping,
-  saveBankPreset, loadBankPresets,
+  saveBankPreset, loadBankPresets, splitByCoverage, inFileDuplicates,
   type DateFormat, type ColumnMapping, type ParsedTransaction, type BankPreset,
 } from "@/lib/csv-parser";
 
@@ -14,6 +14,8 @@ const TX_TYPES = ["debit", "credit", "payment", "refund", "fee", "interest", "tr
 
 type Step = "upload" | "map" | "review" | "done";
 
+
+
 function ColSelect({
   label, value, onChange, options, required,
 }: {
@@ -54,6 +56,9 @@ export function CsvImportModal({ onClose }: { onClose: () => void }) {
   const [error, setError] = useState("");
   const [presets, setPresets] = useState<BankPreset[]>([]);
   const [insertedCount, setInsertedCount] = useState(0);
+  const [coveredRows, setCoveredRows] = useState<ParsedTransaction[]>([]);
+  const [showCovered, setShowCovered] = useState(false);
+  const coverage = useStatementCoverage();
 
   useEffect(() => { setPresets(loadBankPresets()); }, []);
 
@@ -111,7 +116,9 @@ export function CsvImportModal({ onClose }: { onClose: () => void }) {
     if (savePreset) {
       saveBankPreset({ bankName: bankName.trim(), mapping, dateFormat });
     }
-    setEditedRows(parsed);
+    const { keep, covered } = splitByCoverage(parsed, coverage.data ?? []);
+    setEditedRows(keep);
+    setCoveredRows(covered);
     setStep("review");
   }
 
@@ -120,7 +127,13 @@ export function CsvImportModal({ onClose }: { onClose: () => void }) {
     const valid = editedRows.filter((r) => r.date && r.amount > 0 && r.description);
     if (!valid.length) { setError("No valid rows to import"); return; }
     try {
-      const result = await importCSV.mutateAsync({ bank_name: bankName, transactions: valid });
+      const result = await importCSV.mutateAsync({
+        bank_name: bankName,
+        // Provenance, so a second import of the same file inserts nothing —
+        // paired with each row's source_ref where the file carries an id.
+        source: bankName.trim().toLowerCase().replace(/\s+/g, "-"),
+        transactions: valid,
+      });
       setInsertedCount(result.inserted);
       setStep("done");
     } catch (e) {
@@ -286,7 +299,14 @@ export function CsvImportModal({ onClose }: { onClose: () => void }) {
               <div className="grid grid-cols-2 gap-3">
                 <ColSelect label="Merchant Column (optional)" value={mapping.merchantCol ?? ""} onChange={(v) => setMapping((m) => ({ ...m, merchantCol: v || undefined }))} options={columnLabels} />
                 <ColSelect label="Category Column (optional)" value={mapping.categoryCol ?? ""} onChange={(v) => setMapping((m) => ({ ...m, categoryCol: v || undefined }))} options={columnLabels} />
+                <ColSelect label="Account Column (optional)" value={mapping.accountCol ?? ""} onChange={(v) => setMapping((m) => ({ ...m, accountCol: v || undefined }))} options={columnLabels} />
+                <ColSelect label="Row ID Column (optional)" value={mapping.sourceRefCol ?? ""} onChange={(v) => setMapping((m) => ({ ...m, sourceRefCol: v || undefined }))} options={columnLabels} />
               </div>
+              <p className="text-[11px] text-zinc-500 leading-relaxed">
+                Map <b>Account</b> when the file covers more than one account: rows already
+                covered by that account&apos;s statements are then left out. Map <b>Row ID</b> to
+                make re-importing the same file do nothing.
+              </p>
 
               <label className="flex items-center gap-2 text-sm cursor-pointer text-zinc-400">
                 <input type="checkbox" checked={savePreset} onChange={(e) => setSavePreset(e.target.checked)} className="accent-indigo-500" />
@@ -300,9 +320,52 @@ export function CsvImportModal({ onClose }: { onClose: () => void }) {
           {/* Step 3: Review */}
           {step === "review" && (
             <div>
-              <p className="text-xs text-zinc-500 mb-3">
-                {editedRows.length} transactions parsed. Edit or remove rows before importing.
+              <p className="text-xs text-zinc-500 mb-2">
+                {editedRows.length} transactions to import. Edit or remove rows before importing.
               </p>
+
+              {/*
+                Reported, never silent. A row that disappears can mean several
+                things and one of them is worth following up, so the count is
+                always shown and the rows themselves are one click away.
+              */}
+              {coveredRows.length > 0 && (
+                <div className="mb-3 rounded border border-zinc-800 bg-zinc-800/30 px-3 py-2">
+                  <button
+                    type="button"
+                    onClick={() => setShowCovered((v) => !v)}
+                    className="text-xs text-zinc-300 hover:text-white text-left w-full"
+                  >
+                    {coveredRows.length} row{coveredRows.length === 1 ? "" : "s"} left out — already covered by
+                    a statement for that account.{" "}
+                    <span className="text-indigo-400">{showCovered ? "hide" : "show"}</span>
+                  </button>
+                  {showCovered && (
+                    <>
+                      <ul className="mt-2 max-h-40 overflow-y-auto text-[11px] text-zinc-500 font-mono space-y-0.5">
+                        {coveredRows.slice(0, 200).map((r, i) => (
+                          <li key={i}>{r.date}  {r.amount.toFixed(2).padStart(10)}  {r.description.slice(0, 44)}</li>
+                        ))}
+                      </ul>
+                      {coveredRows.length > 200 && (
+                        <p className="mt-1 text-[11px] text-zinc-600">…and {coveredRows.length - 200} more.</p>
+                      )}
+                      <p className="mt-2 text-[11px] text-zinc-500">
+                        Statements cover:{" "}
+                        {(coverage.data ?? []).map((c) => `${c.last4} to ${c.coveredTo}`).join(" · ")}
+                      </p>
+                    </>
+                  )}
+                </div>
+              )}
+
+              {inFileDuplicates(editedRows) > 0 && (
+                <p className="mb-3 rounded border border-amber-900/50 bg-amber-950/30 px-3 py-2 text-xs text-amber-300">
+                  {inFileDuplicates(editedRows)} row{inFileDuplicates(editedRows) === 1 ? " duplicates another row" : "s duplicate other rows"} in
+                  this same file (same date, amount and description). A re-consent can make an
+                  aggregator re-export history under new ids — check before importing.
+                </p>
+              )}
               <div className="overflow-x-auto rounded border border-zinc-800">
                 <table className="w-full text-xs">
                   <thead className="border-b border-zinc-800">