Add /api/frollo/ingest so the import can run unattended
ci / lint-test (push) Successful in 40s

Shares one module with the CLI rather than reimplementing the insert:
frollo-ingest.ts holds parsing, scoping, de-duplication and the write, and
both callers pass in their own SQL executor (Prisma in the route, a pg
client in the script). The alternative is two implementations of the same
insert, which is how the pantry healthcheck came to be fixed in one repo
and left broken in the other.

The route refuses rather than guesses. findAnomalies() returns every reason
an unattended run should stop - a configured account contributing no rows,
an unrecognised account, a near-consecutive-id collapse that might be a
real repeat, a batch over ~200 rows, or an export taken with pending
included - and the route answers 409 having written nothing.

Two defects the wiring surfaced. Deliberately excluded credit cards were
reported as unknown accounts, which would have raised the new-account
anomaly on every single run and left the automatic path permanently
refusing; EXCLUDED_ACCOUNTS now distinguishes excluded from unknown. And
pending was tested after account scope, so pending rows on cards - which is
all of them so far - classified as out-of-scope and the wrong-export-option
signal could never fire; pending is now tested first.
This commit is contained in:
2026-08-13 11:19:27 +10:00
parent 493ff6f631
commit 21e9e765a3
5 changed files with 455 additions and 221 deletions
+79
View File
@@ -2,6 +2,8 @@ import { describe, it, expect } from "vitest";
import { parseCSVRows } from "@/lib/csv-parser";
import {
ACCOUNTS,
EXCLUDED_ACCOUNTS,
isKnownExcluded,
SUSPECT_REPEAT_GAP,
accountFor,
accountReport,
@@ -14,6 +16,7 @@ import {
unknownAccounts,
type FrolloRow,
} from "@/lib/frollo-csv";
import { LARGE_BATCH, findAnomalies } from "@/lib/frollo-ingest";
const HEADER =
"transaction_id,description,user_description,amount,currency,transaction_date,posted_date," +
@@ -307,3 +310,79 @@ describe("accountReport / unknownAccounts", () => {
expect(unknown[0]).toMatchObject({ accountName: "New Card", rows: 1 });
});
});
describe("findAnomalies", () => {
const base = { missingAccounts: [], unknownAccounts: [], suspectRepeats: [], toInsert: 10, skipped: {} };
it("passes a boring run", () => {
expect(findAnomalies(base)).toEqual([]);
});
it("flags a configured account that contributed nothing — a lapsed consent", () => {
const out = findAnomalies({ ...base, missingAccounts: ["4830 Wise (income)"] });
expect(out).toHaveLength(1);
expect(out[0]).toMatch(/consent may have lapsed/i);
});
it("flags an account that appeared in the file unannounced", () => {
const out = findAnomalies({
...base,
unknownAccounts: [{ accountNumber: "xxxx9999", accountName: "New Everyday", rows: 4 }],
});
expect(out[0]).toMatch(/unknown account/i);
});
it("flags a de-duplication that might have collapsed a real repeat", () => {
const out = findAnomalies({
...base,
suspectRepeats: [{ description: "X", transactionDate: "2026-01-01", amount: -1, idGap: 2 }],
});
expect(out[0]).toMatch(/genuine repeats/i);
});
it("flags a batch far larger than an ordinary month", () => {
expect(findAnomalies({ ...base, toInsert: LARGE_BATCH + 1 })).toHaveLength(1);
expect(findAnomalies({ ...base, toInsert: LARGE_BATCH })).toHaveLength(0);
});
it("flags an export taken with pending included", () => {
const out = findAnomalies({ ...base, skipped: { pending: 3 } });
expect(out[0]).toMatch(/pending/i);
});
it("reports every anomaly at once rather than the first", () => {
expect(
findAnomalies({ ...base, missingAccounts: ["a"], toInsert: LARGE_BATCH + 1, skipped: { pending: 1 } })
).toHaveLength(3);
});
});
describe("excluded vs unknown accounts", () => {
it("does not report a deliberately excluded credit card as unknown", () => {
// An alert that fires on every run is an alert nobody reads.
const rows = csv(line({ id: "1", amount: "-50.00", account: "xxxx xxxx xxxx 3893", accountName: "Ultimate Awards credit card" }));
expect(unknownAccounts(rows)).toEqual([]);
expect(isKnownExcluded("xxxx xxxx xxxx 3893")).toBe(true);
});
it("still reports a genuinely new account", () => {
const rows = csv(line({ id: "1", amount: "-50.00", account: "xxxx xxxx xxxx 9999", accountName: "Brand New" }));
expect(unknownAccounts(rows)).toHaveLength(1);
});
it("keeps the imported and excluded rosters disjoint", () => {
const a = new Set(ACCOUNTS.map((x) => x.last4));
for (const e of EXCLUDED_ACCOUNTS) expect(a.has(e.last4)).toBe(false);
});
});
describe("pending detection precedes account scoping", () => {
it("reports a pending credit-card row as pending, not out of scope", () => {
// Otherwise the wrong-export-option signal never fires, since every pending
// row observed so far has been on a card.
const row = csv(
line({ id: "1", amount: "-13.50", desc: "Pending - Breadworld Bakery Cafe", account: "xxxx xxxx xxxx 8032" })
)[0];
expect(skipReason(row)).toBe("pending");
});
});