Add /api/frollo/ingest so the import can run unattended

Shares one module with the CLI rather than reimplementing the insert: frollo-ingest.ts holds parsing, scoping, de-duplication and the write, and both callers pass in their own SQL executor (Prisma in the route, a pg client in the script). The alternative is two implementations of the same insert, which is how the pantry healthcheck came to be fixed in one repo and left broken in the other. The route refuses rather than guesses. findAnomalies() returns every reason an unattended run should stop - a configured account contributing no rows, an unrecognised account, a near-consecutive-id collapse that might be a real repeat, a batch over ~200 rows, or an export taken with pending included - and the route answers 409 having written nothing. Two defects the wiring surfaced. Deliberately excluded credit cards were reported as unknown accounts, which would have raised the new-account anomaly on every single run and left the automatic path permanently refusing; EXCLUDED_ACCOUNTS now distinguishes excluded from unknown. And pending was tested after account scope, so pending rows on cards - which is all of them so far - classified as out-of-scope and the wrong-export-option signal could never fire; pending is now tested first.
2026-08-13 11:19:27 +10:00
parent 493ff6f631
commit 21e9e765a3
5 changed files with 455 additions and 221 deletions
@@ -97,6 +97,30 @@ export const ACCOUNTS: AccountSpec[] = [
  { last4: "9807", label: "ANZ Online Saver", provider: "ANZ" },
 ];

+/**
+ * Accounts knowingly left out, so "excluded" is distinguishable from "unknown".
+ *
+ * Without this the five credit cards read as unrecognised accounts on every
+ * single run, which would raise the new-account anomaly every time and make an
+ * automatic import permanently refuse. An alert that always fires is an alert
+ * nobody reads.
+ *
+ * Anything in neither list is genuinely new — a freshly connected account, or a
+ * re-masked number — and does deserve a look before its rows are ignored.
+ */
+export const EXCLUDED_ACCOUNTS: AccountSpec[] = [
+  { last4: "3893", label: "CommBank Ultimate Awards (card)", provider: "CommBank" },
+  { last4: "8032", label: "Westpac Altitude Qantas Black (card)", provider: "Westpac" },
+  { last4: "1000", label: "Amex Platinum Business (card)", provider: "American Express" },
+  { last4: "0351", label: "HSBC Star Alliance (card)", provider: "HSBC" },
+  { last4: "6227", label: "MyCard Prestige (card)", provider: "Mycard" },
+];
+
+export function isKnownExcluded(accountNumber: string): boolean {
+  const key = last4(accountNumber);
+  return EXCLUDED_ACCOUNTS.some((a) => a.last4 === key);
+}
+
 /**
 * Below this id gap, two rows sharing a natural key are *probably* a genuine
 * same-day repeat rather than a re-ingest twin — and are reported rather than
@@ -196,8 +220,13 @@ export type SkipReason =
 export function skipReason(row: FrolloRow): SkipReason | null {
  if (!Number.isFinite(row.amount)) return "unparseable_amount";
  if (row.amount === 0) return "zero_amount";
-  if (!accountFor(row.accountNumber)) return "out_of_scope_account";
+  // Pending is tested before account scope on purpose. Every pending row seen so
+  // far has been on a credit card, so scoping first would classify them all as
+  // out-of-scope and the "this export was taken with pending INCLUDED" signal
+  // would never fire — even though it is worth acting on wherever it appears,
+  // because the option makes successive exports incomparable.
  if (/^\s*pending\s*[-:]/i.test(row.description)) return "pending";
+  if (!accountFor(row.accountNumber)) return "out_of_scope_account";
  return null;
 }

@@ -333,11 +362,17 @@ export function accountReport(rows: FrolloRow[]): AccountReport[] {
  }));
 }

-/** Accounts in the file that this importer does not know about. */
+/**
+ * Accounts in the file that are neither imported nor knowingly excluded.
+ *
+ * Empty is the normal state. A name appearing here means a new account reached
+ * the export and its rows are being dropped on the floor — worth a look before
+ * that becomes the status quo.
+ */
 export function unknownAccounts(rows: FrolloRow[]): { accountNumber: string; accountName: string; rows: number }[] {
  const seen = new Map<string, { accountNumber: string; accountName: string; rows: number }>();
  for (const r of rows) {
-    if (accountFor(r.accountNumber)) continue;
+    if (accountFor(r.accountNumber) || isKnownExcluded(r.accountNumber)) continue;
    const k = last4(r.accountNumber);
    const e = seen.get(k);
    if (e) e.rows++;
@@ -0,0 +1,201 @@
+import {
+  accountReport,
+  dedupe,
+  readRows,
+  skipReason,
+  toLedgerRow,
+  unknownAccounts,
+  type AccountReport,
+  type FrolloRow,
+  type SkipReason,
+} from "./frollo-csv.ts";
+import { parseCSVRows } from "./csv-parser.ts";
+
+/**
+ * Turns a Frollo CSV into ledger rows and (optionally) writes them.
+ *
+ * Shared by `scripts/import-frollo.mts` and `POST /api/frollo/ingest` so the
+ * hand-run and automatic paths cannot drift. They previously would have been two
+ * implementations of the same insert, which is exactly how the pantry healthcheck
+ * came to be fixed in one repo and left broken in the other.
+ *
+ * The database is reached through an injected executor rather than imported
+ * directly: the API route runs inside Next with Prisma, while the CLI runs under
+ * `node --experimental-strip-types` with a plain `pg` client and no path aliases.
+ */
+// The .ts extensions above are load-bearing: this module is imported both by
+// Next (which resolves either form) and by scripts/import-frollo.mts running
+// under `node --experimental-strip-types`, whose ESM resolver requires the
+// extension written out. tsconfig sets allowImportingTsExtensions for this.
+export type SqlExecutor = <T = unknown>(sql: string, params: unknown[]) => Promise<T[]>;
+
+export const SOURCE = "frollo";
+
+/**
+ * Above this many new rows, an automatic run stops and asks.
+ *
+ * Steady state is ~50 rows a month. A batch several times that means something
+ * changed — a re-consent duplicating history, a longer export window, or a new
+ * account — and none of those should land unseen. The first (backfill) import
+ * was 550 and was run by hand with `force`.
+ */
+export const LARGE_BATCH = 200;
+
+export interface IngestOptions {
+  ownerId?: number;
+  /** Write. Without it, everything is computed and nothing is inserted. */
+  apply?: boolean;
+  /** Proceed despite anomalies. For a human who has read them. */
+  force?: boolean;
+  largeBatch?: number;
+}
+
+export interface IngestReport {
+  totalRows: number;
+  inScope: number;
+  skipped: Partial<Record<SkipReason, number>>;
+  accounts: AccountReport[];
+  missingAccounts: string[];
+  unknownAccounts: { accountNumber: string; accountName: string; rows: number }[];
+  duplicatesDropped: number;
+  suspectRepeats: { description: string; transactionDate: string; amount: number; idGap: number }[];
+  alreadyImported: number;
+  toInsert: number;
+  inserted: number;
+  /** Non-empty means an automatic run should stop and a human should look. */
+  anomalies: string[];
+  applied: boolean;
+}
+
+/**
+ * Reasons an automatic import should stop rather than write.
+ *
+ * Pure, so it is testable without a database. Every entry is a thing that has
+ * either happened or come one verification away from happening: a lapsed CDR
+ * consent silently shrinking the file, a new account appearing unnoticed, a
+ * de-duplication that might have collapsed a real repeat, and a batch far larger
+ * than the account activity can explain.
+ */
+export function findAnomalies(
+  report: Pick<IngestReport, "missingAccounts" | "unknownAccounts" | "suspectRepeats" | "toInsert" | "skipped">,
+  largeBatch = LARGE_BATCH
+): string[] {
+  const out: string[] = [];
+  if (report.missingAccounts.length > 0) {
+    out.push(
+      `${report.missingAccounts.length} configured account(s) contributed no rows ` +
+      `(${report.missingAccounts.join(", ")}) — a CDR consent may have lapsed`
+    );
+  }
+  if (report.unknownAccounts.length > 0) {
+    out.push(
+      `${report.unknownAccounts.length} unknown account(s) in the file, skipped ` +
+      `(${report.unknownAccounts.map((a) => a.accountName).join(", ")})`
+    );
+  }
+  if (report.suspectRepeats.length > 0) {
+    out.push(
+      `${report.suspectRepeats.length} de-duplicated row(s) had near-consecutive ids ` +
+      `and may be genuine repeats rather than twins`
+    );
+  }
+  if (report.toInsert > largeBatch) {
+    out.push(`${report.toInsert} new rows is more than the expected ${largeBatch}`);
+  }
+  if ((report.skipped.pending ?? 0) > 0) {
+    out.push(
+      `${report.skipped.pending} pending row(s) present — export was taken with pending ` +
+      `INCLUDED; they are skipped, but re-export with it excluded`
+    );
+  }
+  return out;
+}
+
+export async function ingestFrolloCsv(
+  csvText: string,
+  exec: SqlExecutor,
+  opts: IngestOptions = {}
+): Promise<IngestReport> {
+  const ownerId = opts.ownerId ?? 1;
+  const rows = readRows(parseCSVRows(csvText));
+
+  const skipped: Partial<Record<SkipReason, number>> = {};
+  const inScope: FrolloRow[] = [];
+  for (const r of rows) {
+    const why = skipReason(r);
+    if (why) skipped[why] = (skipped[why] ?? 0) + 1;
+    else inScope.push(r);
+  }
+
+  const accounts = accountReport(inScope);
+  const missingAccounts = accounts.filter((a) => !a.present).map((a) => `${a.spec.last4} ${a.spec.label}`);
+  const unknown = unknownAccounts(rows);
+
+  const { kept, dropped } = dedupe(inScope);
+  const suspectRepeats = dropped
+    .filter((d) => d.suspectGenuineRepeat)
+    .map((d) => ({
+      description: d.row.description,
+      transactionDate: d.row.transactionDate,
+      amount: d.row.amount,
+      idGap: d.idGap,
+    }));
+
+  const ledger = kept.map(toLedgerRow);
+  const existing = await exec<{ source_ref: string }>(
+    "SELECT source_ref FROM transactions WHERE source = $1",
+    [SOURCE]
+  );
+  const seen = new Set(existing.map((r) => r.source_ref));
+  const fresh = ledger.filter((r) => !seen.has(r.sourceRef));
+
+  const base = {
+    totalRows: rows.length,
+    inScope: inScope.length,
+    skipped,
+    accounts,
+    missingAccounts,
+    unknownAccounts: unknown,
+    duplicatesDropped: dropped.length,
+    suspectRepeats,
+    alreadyImported: ledger.length - fresh.length,
+    toInsert: fresh.length,
+  };
+  const anomalies = findAnomalies(base, opts.largeBatch);
+
+  if (!opts.apply || (anomalies.length > 0 && !opts.force)) {
+    return { ...base, inserted: 0, anomalies, applied: false };
+  }
+
+  let inserted = 0;
+  for (const r of fresh) {
+    // ON CONFLICT against uq_transaction_source_ref: belt and braces over the
+    // source_ref check above, which is not transactional.
+    const res = await exec<{ id: number }>(
+      `INSERT INTO transactions
+         (statement_id, owner_id, transaction_date, description, amount, transaction_type,
+          merchant_name, foreign_currency_amount, foreign_currency_code,
+          source, source_ref, source_account)
+       VALUES (NULL, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
+       ON CONFLICT (source, source_ref) WHERE source IS NOT NULL AND source_ref IS NOT NULL
+       DO NOTHING
+       RETURNING id`,
+      [
+        ownerId,
+        r.transactionDate,
+        r.description,
+        r.amount,
+        r.transactionType,
+        r.merchantName,
+        r.foreignCurrencyAmount,
+        r.foreignCurrencyCode,
+        SOURCE,
+        r.sourceRef,
+        r.sourceAccount,
+      ]
+    );
+    inserted += res.length;
+  }
+
+  return { ...base, inserted, anomalies, applied: true };
+}