Shares one module with the CLI rather than reimplementing the insert: frollo-ingest.ts holds parsing, scoping, de-duplication and the write, and both callers pass in their own SQL executor (Prisma in the route, a pg client in the script). The alternative is two implementations of the same insert, which is how the pantry healthcheck came to be fixed in one repo and left broken in the other. The route refuses rather than guesses. findAnomalies() returns every reason an unattended run should stop - a configured account contributing no rows, an unrecognised account, a near-consecutive-id collapse that might be a real repeat, a batch over ~200 rows, or an export taken with pending included - and the route answers 409 having written nothing. Two defects the wiring surfaced. Deliberately excluded credit cards were reported as unknown accounts, which would have raised the new-account anomaly on every single run and left the automatic path permanently refusing; EXCLUDED_ACCOUNTS now distinguishes excluded from unknown. And pending was tested after account scope, so pending rows on cards - which is all of them so far - classified as out-of-scope and the wrong-export-option signal could never fire; pending is now tested first.
This commit is contained in:
@@ -0,0 +1,201 @@
|
||||
import {
|
||||
accountReport,
|
||||
dedupe,
|
||||
readRows,
|
||||
skipReason,
|
||||
toLedgerRow,
|
||||
unknownAccounts,
|
||||
type AccountReport,
|
||||
type FrolloRow,
|
||||
type SkipReason,
|
||||
} from "./frollo-csv.ts";
|
||||
import { parseCSVRows } from "./csv-parser.ts";
|
||||
|
||||
/**
|
||||
* Turns a Frollo CSV into ledger rows and (optionally) writes them.
|
||||
*
|
||||
* Shared by `scripts/import-frollo.mts` and `POST /api/frollo/ingest` so the
|
||||
* hand-run and automatic paths cannot drift. They previously would have been two
|
||||
* implementations of the same insert, which is exactly how the pantry healthcheck
|
||||
* came to be fixed in one repo and left broken in the other.
|
||||
*
|
||||
* The database is reached through an injected executor rather than imported
|
||||
* directly: the API route runs inside Next with Prisma, while the CLI runs under
|
||||
* `node --experimental-strip-types` with a plain `pg` client and no path aliases.
|
||||
*/
|
||||
// The .ts extensions above are load-bearing: this module is imported both by
|
||||
// Next (which resolves either form) and by scripts/import-frollo.mts running
|
||||
// under `node --experimental-strip-types`, whose ESM resolver requires the
|
||||
// extension written out. tsconfig sets allowImportingTsExtensions for this.
|
||||
export type SqlExecutor = <T = unknown>(sql: string, params: unknown[]) => Promise<T[]>;
|
||||
|
||||
export const SOURCE = "frollo";
|
||||
|
||||
/**
|
||||
* Above this many new rows, an automatic run stops and asks.
|
||||
*
|
||||
* Steady state is ~50 rows a month. A batch several times that means something
|
||||
* changed — a re-consent duplicating history, a longer export window, or a new
|
||||
* account — and none of those should land unseen. The first (backfill) import
|
||||
* was 550 and was run by hand with `force`.
|
||||
*/
|
||||
export const LARGE_BATCH = 200;
|
||||
|
||||
export interface IngestOptions {
|
||||
ownerId?: number;
|
||||
/** Write. Without it, everything is computed and nothing is inserted. */
|
||||
apply?: boolean;
|
||||
/** Proceed despite anomalies. For a human who has read them. */
|
||||
force?: boolean;
|
||||
largeBatch?: number;
|
||||
}
|
||||
|
||||
export interface IngestReport {
|
||||
totalRows: number;
|
||||
inScope: number;
|
||||
skipped: Partial<Record<SkipReason, number>>;
|
||||
accounts: AccountReport[];
|
||||
missingAccounts: string[];
|
||||
unknownAccounts: { accountNumber: string; accountName: string; rows: number }[];
|
||||
duplicatesDropped: number;
|
||||
suspectRepeats: { description: string; transactionDate: string; amount: number; idGap: number }[];
|
||||
alreadyImported: number;
|
||||
toInsert: number;
|
||||
inserted: number;
|
||||
/** Non-empty means an automatic run should stop and a human should look. */
|
||||
anomalies: string[];
|
||||
applied: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reasons an automatic import should stop rather than write.
|
||||
*
|
||||
* Pure, so it is testable without a database. Every entry is a thing that has
|
||||
* either happened or come one verification away from happening: a lapsed CDR
|
||||
* consent silently shrinking the file, a new account appearing unnoticed, a
|
||||
* de-duplication that might have collapsed a real repeat, and a batch far larger
|
||||
* than the account activity can explain.
|
||||
*/
|
||||
export function findAnomalies(
|
||||
report: Pick<IngestReport, "missingAccounts" | "unknownAccounts" | "suspectRepeats" | "toInsert" | "skipped">,
|
||||
largeBatch = LARGE_BATCH
|
||||
): string[] {
|
||||
const out: string[] = [];
|
||||
if (report.missingAccounts.length > 0) {
|
||||
out.push(
|
||||
`${report.missingAccounts.length} configured account(s) contributed no rows ` +
|
||||
`(${report.missingAccounts.join(", ")}) — a CDR consent may have lapsed`
|
||||
);
|
||||
}
|
||||
if (report.unknownAccounts.length > 0) {
|
||||
out.push(
|
||||
`${report.unknownAccounts.length} unknown account(s) in the file, skipped ` +
|
||||
`(${report.unknownAccounts.map((a) => a.accountName).join(", ")})`
|
||||
);
|
||||
}
|
||||
if (report.suspectRepeats.length > 0) {
|
||||
out.push(
|
||||
`${report.suspectRepeats.length} de-duplicated row(s) had near-consecutive ids ` +
|
||||
`and may be genuine repeats rather than twins`
|
||||
);
|
||||
}
|
||||
if (report.toInsert > largeBatch) {
|
||||
out.push(`${report.toInsert} new rows is more than the expected ${largeBatch}`);
|
||||
}
|
||||
if ((report.skipped.pending ?? 0) > 0) {
|
||||
out.push(
|
||||
`${report.skipped.pending} pending row(s) present — export was taken with pending ` +
|
||||
`INCLUDED; they are skipped, but re-export with it excluded`
|
||||
);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function ingestFrolloCsv(
|
||||
csvText: string,
|
||||
exec: SqlExecutor,
|
||||
opts: IngestOptions = {}
|
||||
): Promise<IngestReport> {
|
||||
const ownerId = opts.ownerId ?? 1;
|
||||
const rows = readRows(parseCSVRows(csvText));
|
||||
|
||||
const skipped: Partial<Record<SkipReason, number>> = {};
|
||||
const inScope: FrolloRow[] = [];
|
||||
for (const r of rows) {
|
||||
const why = skipReason(r);
|
||||
if (why) skipped[why] = (skipped[why] ?? 0) + 1;
|
||||
else inScope.push(r);
|
||||
}
|
||||
|
||||
const accounts = accountReport(inScope);
|
||||
const missingAccounts = accounts.filter((a) => !a.present).map((a) => `${a.spec.last4} ${a.spec.label}`);
|
||||
const unknown = unknownAccounts(rows);
|
||||
|
||||
const { kept, dropped } = dedupe(inScope);
|
||||
const suspectRepeats = dropped
|
||||
.filter((d) => d.suspectGenuineRepeat)
|
||||
.map((d) => ({
|
||||
description: d.row.description,
|
||||
transactionDate: d.row.transactionDate,
|
||||
amount: d.row.amount,
|
||||
idGap: d.idGap,
|
||||
}));
|
||||
|
||||
const ledger = kept.map(toLedgerRow);
|
||||
const existing = await exec<{ source_ref: string }>(
|
||||
"SELECT source_ref FROM transactions WHERE source = $1",
|
||||
[SOURCE]
|
||||
);
|
||||
const seen = new Set(existing.map((r) => r.source_ref));
|
||||
const fresh = ledger.filter((r) => !seen.has(r.sourceRef));
|
||||
|
||||
const base = {
|
||||
totalRows: rows.length,
|
||||
inScope: inScope.length,
|
||||
skipped,
|
||||
accounts,
|
||||
missingAccounts,
|
||||
unknownAccounts: unknown,
|
||||
duplicatesDropped: dropped.length,
|
||||
suspectRepeats,
|
||||
alreadyImported: ledger.length - fresh.length,
|
||||
toInsert: fresh.length,
|
||||
};
|
||||
const anomalies = findAnomalies(base, opts.largeBatch);
|
||||
|
||||
if (!opts.apply || (anomalies.length > 0 && !opts.force)) {
|
||||
return { ...base, inserted: 0, anomalies, applied: false };
|
||||
}
|
||||
|
||||
let inserted = 0;
|
||||
for (const r of fresh) {
|
||||
// ON CONFLICT against uq_transaction_source_ref: belt and braces over the
|
||||
// source_ref check above, which is not transactional.
|
||||
const res = await exec<{ id: number }>(
|
||||
`INSERT INTO transactions
|
||||
(statement_id, owner_id, transaction_date, description, amount, transaction_type,
|
||||
merchant_name, foreign_currency_amount, foreign_currency_code,
|
||||
source, source_ref, source_account)
|
||||
VALUES (NULL, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
ON CONFLICT (source, source_ref) WHERE source IS NOT NULL AND source_ref IS NOT NULL
|
||||
DO NOTHING
|
||||
RETURNING id`,
|
||||
[
|
||||
ownerId,
|
||||
r.transactionDate,
|
||||
r.description,
|
||||
r.amount,
|
||||
r.transactionType,
|
||||
r.merchantName,
|
||||
r.foreignCurrencyAmount,
|
||||
r.foreignCurrencyCode,
|
||||
SOURCE,
|
||||
r.sourceRef,
|
||||
r.sourceAccount,
|
||||
]
|
||||
);
|
||||
inserted += res.length;
|
||||
}
|
||||
|
||||
return { ...base, inserted, anomalies, applied: true };
|
||||
}
|
||||
Reference in New Issue
Block a user