/** * Imports a Frollo transaction export into `transactions`. * * node --experimental-strip-types scripts/import-frollo.mts --file * node --experimental-strip-types scripts/import-frollo.mts --file --apply * * Dry run by default: it prints exactly what it would do and touches nothing. * Rehearse first — the failure mode here is doubling reported income, and the * de-duplication rule that shipped second only survived because a dry run was * read against the salary rows. * * Export the file with PENDING TRANSACTIONS EXCLUDED. A pending row changes both * its id and its description when it settles, so importing one guarantees a * duplicate on the next run; every pending row observed has been on a credit * card, which this importer does not cover anyway. See DECISIONS.md ING-11. * * All parsing, scoping, de-duplication and insertion live in * `src/lib/frollo-ingest.ts`, shared with POST /api/frollo/ingest so the manual * and automatic paths cannot drift. * * Needs DATABASE_URL. postgres-personal publishes no host port, so from the host: * export DATABASE_URL="postgresql://personal:@$(docker inspect postgres-personal \ * --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'):5432/personal" * The container IP changes on every recreate. */ import { readFileSync } from "node:fs"; import pg from "pg"; import { ingestFrolloCsv, LEDGER_MATCH_DAYS, type SqlExecutor } from "../src/lib/frollo-ingest.ts"; function arg(name: string, fallback?: string): string | undefined { const i = process.argv.indexOf(`--${name}`); if (i >= 0 && process.argv[i + 1] && !process.argv[i + 1].startsWith("--")) return process.argv[i + 1]; return fallback; } const has = (name: string) => process.argv.includes(`--${name}`); const file = arg("file"); if (!file) { console.error("usage: --file [--apply] [--force] [--owner ]"); process.exit(2); } if (!process.env.DATABASE_URL) { console.error("DATABASE_URL is not set — cannot compare against the ledger."); process.exit(2); } const money = (n: number) => n.toLocaleString("en-AU", { minimumFractionDigits: 2, maximumFractionDigits: 2 }); const client = new pg.Client({ connectionString: process.env.DATABASE_URL }); await client.connect(); const exec: SqlExecutor = async (sql: string, params: unknown[]) => (await client.query(sql, params)).rows as T[]; let report; try { report = await ingestFrolloCsv(readFileSync(file, "utf8"), exec, { ownerId: Number(arg("owner", "1")), apply: has("apply"), force: has("force"), }); } finally { // Left open, the process hangs after an exception and looks like a slow query. if (!has("keep-open")) await client.end().catch(() => {}); } console.log(`\n${file}`); console.log(` ${report.totalRows} rows in file\n`); console.log("SCOPE"); for (const [why, n] of Object.entries(report.skipped).sort((a, b) => b[1]! - a[1]!)) { console.log(` skipped ${String(n).padStart(5)} ${why}`); } console.log(` in scope ${String(report.inScope).padStart(4)}\n`); console.log("ACCOUNTS"); for (const a of report.accounts) { console.log( ` ${a.present ? " " : "!"} ${a.spec.last4} ${a.spec.label.padEnd(24)} ${String(a.rows).padStart(5)} rows` ); } if (report.unknownAccounts.length > 0) { console.log("\n accounts in the file this importer does not know (skipped):"); for (const u of report.unknownAccounts) { console.log(` ${u.accountNumber} ${u.accountName} ${u.rows} rows`); } } console.log(); console.log("DE-DUPLICATION (re-ingest twins from CDR re-consent)"); console.log(` ${report.duplicatesDropped} dropped`); if (report.suspectRepeats.length > 0) { console.log(`\n REVIEW: ${report.suspectRepeats.length} collapsed row(s) had near-consecutive ids`); console.log(" and may be real repeats rather than duplicates:"); for (const s of report.suspectRepeats.slice(0, 10)) { console.log(` ${s.transactionDate} ${money(s.amount).padStart(11)} ${s.description.slice(0, 40)} (gap ${s.idGap})`); } } console.log(); console.log("LEDGER"); console.log(` already imported : ${String(report.alreadyImported).padStart(5)}`); // Printed even when zero. This is the count that was missing on 2026-08-13, // when the whole file was written on top of a ledger that already held 77% of // it; a number you have to go looking for is a number nobody looks at. console.log(` already on ledger: ${String(report.ledgerDuplicates).padStart(5)} (statement twin within ${LEDGER_MATCH_DAYS} days)`); console.log(` new to insert : ${String(report.toInsert).padStart(5)}`); if (report.anomalies.length > 0) { console.log("\nANOMALIES"); for (const a of report.anomalies) console.log(` ! ${a}`); } if (report.applied) { console.log(`\nInserted ${report.inserted} row(s).\n`); } else if (report.anomalies.length > 0 && has("apply")) { console.log("\nNOT APPLIED — anomalies above. Read them, then re-run with --force.\n"); process.exit(1); } else { console.log("\nDRY RUN — nothing written. Re-run with --apply to insert.\n"); }