Accept grocery receipts scanned in pantry as candidate spend
ci / lint-test (push) Successful in 48s
ci / lint-test (push) Successful in 48s
Adds /api/receipts/ingest as a sibling to the order lane, sharing its shape but making one decision differently: nothing is parked. An order can wait for its statement because it is already visible as an email; a gift-card grocery shop is visible nowhere at all, so a scan that produces no transaction produces nothing anyone can see. Every payment becomes a manual row immediately and the existing pending-reconciliation queue resolves the ones with a card leg coming. One transaction per tender leg. A $114.57 shop settled $40.75 gift card + $73.82 Mastercard has a statement line for $73.82 only. A single row marked credits is excluded from the queue while that line double-counts; marked card it is searched for at 1% of $114.57 and never matches. Either way the shop books $188.39. Per-leg rows make each amount the settled amount, so the matcher works untouched. Reconciliation now carries expense_metadata across. It already moved overrides, tags and splits from the manual row to the statement row and left metadata behind, which did not matter while metadata only came from an email that made its own transaction. It matters now that it carries a shop's line items: unmoved, the contents vanish at exactly the moment the statement line appears, and COLES 0556 MANOR LAKES stays as unreadable as before anything was scanned. transaction_id is UNIQUE, so a statement row that already has metadata keeps it and the pantry row is flagged rather than raising a constraint violation. Also regenerates the Prisma model. card_last4, currency, flags, reconciled_at, matched_transaction_id, platform and route have been in the database since migrations 0019/0020 and were absent from schema.prisma — regenerating the client from it would have dropped columns the order lane writes on every ingest. 23 integration tests against the real schema, built from the three receipts that drove the design. Existing suites unchanged: 104 unit, 144 integration.
This commit is contained in:
@@ -0,0 +1,231 @@
|
||||
import { describe, it, expect, beforeAll, beforeEach } from "vitest";
|
||||
import { Pool } from "pg";
|
||||
import { createPool, resetDB, seedParticipants } from "./helpers";
|
||||
|
||||
/**
|
||||
* The receipt lane, exercised against the real schema.
|
||||
*
|
||||
* Every fixture here is a receipt that actually exists: a Coles split-tender e-receipt
|
||||
* (7777 Werribee, 11/07/2026, $114.57 settled $40.75 gift card + $73.82 Mastercard), a
|
||||
* Woolworths e-receipt (3345 Wyndham Vale, 29/06/2026, $40.75, single gift card) and a
|
||||
* Coles photo (556 Manor Lakes, 29/07/2026, $23.18). The split one is the reason the lane
|
||||
* writes one transaction per tender leg rather than one per receipt, so it is the case
|
||||
* these tests are built around.
|
||||
*/
|
||||
|
||||
const TOKEN = "test-receipt-token";
|
||||
let pool: Pool;
|
||||
let POST: typeof import("../../app/api/receipts/ingest/route").POST;
|
||||
|
||||
const req = (body: unknown, token: string | null = TOKEN) =>
|
||||
({
|
||||
headers: { get: (h: string) => (h === "x-ingest-token" ? token : null) },
|
||||
json: async () => body,
|
||||
}) as never;
|
||||
|
||||
const colesSplit = () => ({
|
||||
receipt_uid: "coles:7777:114:2153:2026-07-11",
|
||||
capture_event_id: 412,
|
||||
image_sha256: "sha-coles-split",
|
||||
merchant_name: "Coles",
|
||||
store_detail: "7777",
|
||||
transaction_date: "2026-07-11",
|
||||
total: 114.57,
|
||||
tender_raw: "EFT $40.75\nEFT $73.82\n***** 0443 MASTERCARD\nCREDIT ACCOUNT STORE CARD",
|
||||
tender_legs: [
|
||||
{ leg_index: 1, amount: 40.75, card_last4: "0443", card_product: "STORE CARD", class: "gift_card" as const },
|
||||
{ leg_index: 2, amount: 73.82, card_last4: "3302", card_product: "MASTERCARD", class: "card" as const },
|
||||
],
|
||||
line_items: [
|
||||
{ name: "Pork Loin Roast", quantity: 1, unit: "ea", line_total: 15.86, category: "meat" },
|
||||
{ name: "Tomatoes", quantity: 1, unit: "ea", line_total: 98.71, category: "produce" },
|
||||
],
|
||||
});
|
||||
|
||||
const woolworthsGiftCard = () => ({
|
||||
receipt_uid: "woolworths:3345:62:148:2026-06-29",
|
||||
capture_event_id: 500,
|
||||
merchant_name: "Woolworths",
|
||||
store_detail: "3345",
|
||||
transaction_date: "2026-06-29",
|
||||
total: 40.75,
|
||||
tender_legs: [{ leg_index: 1, amount: 40.75, card_last4: "0443", card_product: "STORE CARD", class: "gift_card" as const }],
|
||||
line_items: [{ name: "Oat Milk", quantity: 2, unit: "ea", line_total: 40.75, category: "dairy" }],
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
process.env.RECEIPT_INGEST_TOKEN = TOKEN;
|
||||
pool = createPool();
|
||||
({ POST } = await import("../../app/api/receipts/ingest/route"));
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetDB(pool);
|
||||
await pool.query("DELETE FROM expense_metadata");
|
||||
// transactions.owner_id references participants, and resetDB truncates it with RESTART
|
||||
// IDENTITY — so DEFAULT_OWNER_ID (1) has to be re-seeded or every insert here fails the
|
||||
// foreign key. Same dependency the order lane has; it just never had to say so.
|
||||
await seedParticipants(pool);
|
||||
});
|
||||
|
||||
const legsOf = async (group: string) =>
|
||||
(await pool.query(
|
||||
`SELECT t.id, t.amount::text, t.payment_method, t.statement_id, em.order_reference, em.line_items, em.flags
|
||||
FROM expense_metadata em JOIN transactions t ON t.id = em.transaction_id
|
||||
WHERE em.receipt_group = $1 ORDER BY em.order_reference`,
|
||||
[group]
|
||||
)).rows;
|
||||
|
||||
describe("auth", () => {
|
||||
it("rejects a missing or wrong token", async () => {
|
||||
expect((await POST(req(colesSplit(), null))).status).toBe(401);
|
||||
expect((await POST(req(colesSplit(), "nope"))).status).toBe(401);
|
||||
});
|
||||
|
||||
it("rejects a body missing what it needs to book money", async () => {
|
||||
expect((await POST(req({ merchant_name: "Coles" }))).status).toBe(400);
|
||||
});
|
||||
});
|
||||
|
||||
describe("a split-tender receipt", () => {
|
||||
it("books one transaction per leg, not one for the total", async () => {
|
||||
const res = await POST(req(colesSplit()));
|
||||
expect(res.status).toBe(200);
|
||||
const body = await res.json();
|
||||
expect(body.legs).toHaveLength(2);
|
||||
|
||||
const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11");
|
||||
expect(rows.map((r) => r.amount)).toEqual(["40.75", "73.82"]);
|
||||
// Not one $114.57 row: marked credits it would let the $73.82 statement line
|
||||
// double-count; marked card it would be searched for at ±1% of $114.57 and never match.
|
||||
expect(rows.some((r) => r.amount === "114.57")).toBe(false);
|
||||
});
|
||||
|
||||
it("sums the legs back to the printed total", async () => {
|
||||
await POST(req(colesSplit()));
|
||||
const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11");
|
||||
const total = rows.reduce((sum, r) => sum + Number(r.amount), 0);
|
||||
expect(Number(total.toFixed(2))).toBe(114.57);
|
||||
});
|
||||
|
||||
it("marks the gift-card leg credits and the card leg card", async () => {
|
||||
await POST(req(colesSplit()));
|
||||
const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11");
|
||||
expect(rows.find((r) => r.amount === "40.75")!.payment_method).toBe("credits");
|
||||
expect(rows.find((r) => r.amount === "73.82")!.payment_method).toBe("card");
|
||||
});
|
||||
|
||||
it("leaves every leg as a manual row for the reconciliation queue", async () => {
|
||||
await POST(req(colesSplit()));
|
||||
const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11");
|
||||
expect(rows.every((r) => r.statement_id === null)).toBe(true);
|
||||
});
|
||||
|
||||
it("puts the line items on the card leg only", async () => {
|
||||
// They describe the whole shop but transaction_id is UNIQUE on expense_metadata, so
|
||||
// they can attach to one row — and the card leg is the one that reconciles onto the
|
||||
// statement line, which is where an unreadable descriptor gets its contents.
|
||||
await POST(req(colesSplit()));
|
||||
const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11");
|
||||
expect(rows.find((r) => r.amount === "73.82")!.line_items).toHaveLength(2);
|
||||
expect(rows.find((r) => r.amount === "40.75")!.line_items).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("flags the split so it can be shown as one purchase", async () => {
|
||||
await POST(req(colesSplit()));
|
||||
const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11");
|
||||
expect(rows[0].flags).toContain("split_tender");
|
||||
});
|
||||
});
|
||||
|
||||
describe("a gift-card receipt", () => {
|
||||
it("becomes a visible transaction on the day it is scanned", async () => {
|
||||
const res = await POST(req(woolworthsGiftCard()));
|
||||
expect(res.status).toBe(200);
|
||||
const rows = await legsOf("pantry:woolworths:3345:62:148:2026-06-29");
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].amount).toBe("40.75");
|
||||
expect(rows[0].payment_method).toBe("credits");
|
||||
});
|
||||
|
||||
it("is owned, so it is not invisible in every view", async () => {
|
||||
// Owner scoping is COALESCE(t.owner_id, s.owner_id) and these rows carry no statement;
|
||||
// a NULL owner is how 85 backfilled order rows ended up in the table and on no screen.
|
||||
await POST(req(woolworthsGiftCard()));
|
||||
const { rows } = await pool.query("SELECT owner_id FROM transactions WHERE merchant_name = 'Woolworths'");
|
||||
expect(rows[0].owner_id).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("idempotency", () => {
|
||||
it("keys on the receipt, so the same shop from two files lands once", async () => {
|
||||
// A photo and the store's e-receipt PDF are different files with different hashes.
|
||||
// Keying on the capture would let one purchase arrive twice.
|
||||
await POST(req(colesSplit()));
|
||||
await POST(req({ ...colesSplit(), capture_event_id: 999, image_sha256: "sha-different-file" }));
|
||||
const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11");
|
||||
expect(rows).toHaveLength(2); // still just the two legs
|
||||
});
|
||||
|
||||
it("reports the replay rather than silently doing nothing", async () => {
|
||||
await POST(req(colesSplit()));
|
||||
const body = await (await POST(req(colesSplit()))).json();
|
||||
expect(body.legs.every((l: { skipped?: string }) => l.skipped === "already_ingested")).toBe(true);
|
||||
});
|
||||
|
||||
it("falls back to the capture when the receipt did not identify itself", async () => {
|
||||
// A crumpled photo can lose the header entirely. That receipt still becomes spend; it
|
||||
// just cannot be recognised if the same shop is scanned again from another file.
|
||||
await POST(req({ ...colesSplit(), receipt_uid: null }));
|
||||
expect(await legsOf("pantry:capture:412")).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("validation — what becomes money", () => {
|
||||
it("refuses a receipt whose tender does not add up to its total", async () => {
|
||||
// A missed leg is spend that never appears; booking the rest would put a number in the
|
||||
// ledger nobody can stand behind.
|
||||
const broken = { ...colesSplit(), tender_legs: [colesSplit().tender_legs[0]] };
|
||||
const res = await POST(req(broken));
|
||||
expect(res.status).toBe(422);
|
||||
expect((await res.json()).reason).toContain("tender legs sum to 40.75");
|
||||
expect(await legsOf("pantry:coles:7777:114:2153:2026-07-11")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("refuses a receipt with no tender at all", async () => {
|
||||
expect((await POST(req({ ...colesSplit(), tender_legs: [] }))).status).toBe(422);
|
||||
});
|
||||
|
||||
it("flags but accepts lines that do not sum, because promo rows are skipped by design", async () => {
|
||||
const res = await POST(req({ ...colesSplit(), line_items: [{ name: "One line", line_total: 10 }] }));
|
||||
expect(res.status).toBe(200);
|
||||
expect((await res.json()).flags.some((f: string) => f.startsWith("line_items_sum_"))).toBe(true);
|
||||
});
|
||||
|
||||
it("writes nothing at all when validation fails", async () => {
|
||||
await POST(req({ ...colesSplit(), tender_legs: [] }));
|
||||
const { rows } = await pool.query("SELECT count(*)::int AS n FROM transactions");
|
||||
expect(rows[0].n).toBe(0);
|
||||
});
|
||||
|
||||
it("marks a leg nobody could classify as reconcilable rather than deciding for them", async () => {
|
||||
const unknown = { ...colesSplit(), tender_legs: [{ leg_index: 1, amount: 114.57, card_last4: "9999", card_product: null, class: null }] };
|
||||
const res = await POST(req(unknown));
|
||||
expect(res.status).toBe(200);
|
||||
const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11");
|
||||
// NULL is what needsCardMatch() treats as still needing a card match, so it surfaces in
|
||||
// the queue instead of being silently excluded from it.
|
||||
expect(rows[0].payment_method).toBeNull();
|
||||
expect(rows[0].flags).toContain("unclassified_tender");
|
||||
});
|
||||
});
|
||||
|
||||
describe("dry run", () => {
|
||||
it("validates without writing", async () => {
|
||||
const res = await POST(req({ ...colesSplit(), dryRun: true }));
|
||||
expect(res.status).toBe(200);
|
||||
expect((await res.json()).dryRun).toBe(true);
|
||||
const { rows } = await pool.query("SELECT count(*)::int AS n FROM transactions");
|
||||
expect(rows[0].n).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,166 @@
|
||||
import { describe, it, expect, beforeAll, beforeEach, vi } from "vitest";
|
||||
import { Pool } from "pg";
|
||||
import { createPool, resetDB, seedParticipants } from "./helpers";
|
||||
|
||||
/**
|
||||
* What happens to a scanned receipt when its statement finally arrives.
|
||||
*
|
||||
* This is the case the first design of this lane got wrong, so it is tested first-class.
|
||||
* Reconciliation moves a manual row's overrides, tags and splits onto the statement row and
|
||||
* then hides the manual row from every figure. `expense_metadata` was the one child it left
|
||||
* behind — which did not matter while metadata only ever came from an email that had made
|
||||
* its own transaction, and matters completely now that it carries a shop's line items. Left
|
||||
* unmoved, the contents of the shop vanish at exactly the moment the statement line shows
|
||||
* up, and `COLES 0556 MANOR LAKES` stays as unreadable as it was before anything was
|
||||
* scanned.
|
||||
*/
|
||||
|
||||
const TOKEN = "test-receipt-token";
|
||||
let pool: Pool;
|
||||
let ownerId: number;
|
||||
let ingest: typeof import("../../app/api/receipts/ingest/route").POST;
|
||||
let reconcile: typeof import("../../app/api/transactions/reconcile/route").POST;
|
||||
|
||||
const ingestReq = (body: unknown) =>
|
||||
({ headers: { get: (h: string) => (h === "x-ingest-token" ? TOKEN : null) }, json: async () => body }) as never;
|
||||
|
||||
// The reconcile route authenticates a browser session rather than a shared secret.
|
||||
const userReq = (body: unknown) => ({ json: async () => body }) as never;
|
||||
|
||||
const colesSplit = {
|
||||
receipt_uid: "coles:7777:114:2153:2026-07-11",
|
||||
capture_event_id: 412,
|
||||
merchant_name: "Coles",
|
||||
transaction_date: "2026-07-11",
|
||||
total: 114.57,
|
||||
tender_legs: [
|
||||
{ leg_index: 1, amount: 40.75, card_last4: "0443", card_product: "STORE CARD", class: "gift_card" },
|
||||
{ leg_index: 2, amount: 73.82, card_last4: "3302", card_product: "MASTERCARD", class: "card" },
|
||||
],
|
||||
line_items: [
|
||||
{ name: "Pork Loin Roast", quantity: 1, unit: "ea", line_total: 15.86, category: "meat" },
|
||||
{ name: "Jasmine Rice 1kg", quantity: 1, unit: "ea", line_total: 98.71, category: "pantry_dry" },
|
||||
],
|
||||
};
|
||||
|
||||
async function statementLine(amount: number, date: string, description: string) {
|
||||
const statement = await pool.query(
|
||||
`INSERT INTO statements (filename, account_number, bank_name, billing_start_date, billing_end_date, owner_id)
|
||||
VALUES ('sept.pdf', '1234-5678-9012-3302', 'Test Bank', $1::date - 20, $1::date + 10, $2) RETURNING id`,
|
||||
[date, ownerId]
|
||||
);
|
||||
const txn = await pool.query(
|
||||
`INSERT INTO transactions (statement_id, transaction_date, description, amount, transaction_type, owner_id)
|
||||
VALUES ($1, $2, $3, $4, 'debit', $5) RETURNING id`,
|
||||
[statement.rows[0].id, date, description, amount, ownerId]
|
||||
);
|
||||
return txn.rows[0].id as number;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
process.env.RECEIPT_INGEST_TOKEN = TOKEN;
|
||||
pool = createPool();
|
||||
// The reconcile route scopes every query to the signed-in user. Mocked before the import
|
||||
// (doMock is not hoisted, so it can close over `ownerId`, which resetDB reassigns on each
|
||||
// run) because an ES module binding cannot be reassigned afterwards.
|
||||
vi.doMock("@/lib/auth", () => ({
|
||||
getCurrentUser: async () => ({ id: ownerId, name: "Alice", email: "alice@example.com" }),
|
||||
}));
|
||||
({ POST: ingest } = await import("../../app/api/receipts/ingest/route"));
|
||||
({ POST: reconcile } = await import("../../app/api/transactions/reconcile/route"));
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetDB(pool);
|
||||
await pool.query("DELETE FROM expense_metadata");
|
||||
({ ownerId } = await seedParticipants(pool));
|
||||
});
|
||||
|
||||
const reconcileAs = (manualId: number, statementId: number) =>
|
||||
reconcile(userReq({ matches: [{ manual_id: manualId, statement_tx_id: statementId }] }));
|
||||
|
||||
describe("a card leg meeting its statement line", () => {
|
||||
it("carries the shop's line items onto the statement row", async () => {
|
||||
await ingest(ingestReq(colesSplit));
|
||||
const manual = await pool.query(
|
||||
`SELECT t.id FROM transactions t JOIN expense_metadata em ON em.transaction_id = t.id
|
||||
WHERE em.order_reference = 'pantry:coles:7777:114:2153:2026-07-11#2'`
|
||||
);
|
||||
const manualId = manual.rows[0].id as number;
|
||||
const statementId = await statementLine(73.82, "2026-07-12", "COLES 7777 WERRIBEE");
|
||||
|
||||
const res = await reconcileAs(manualId, statementId);
|
||||
expect(res.status ?? 200).toBe(200);
|
||||
|
||||
const moved = await pool.query(`SELECT transaction_id, line_items FROM expense_metadata WHERE order_reference = $1`, [
|
||||
"pantry:coles:7777:114:2153:2026-07-11#2",
|
||||
]);
|
||||
// The whole point: the items are now on the row that survives, not the one that got hidden.
|
||||
expect(moved.rows[0].transaction_id).toBe(statementId);
|
||||
expect(moved.rows[0].line_items).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("counts the shop once, not twice", async () => {
|
||||
await ingest(ingestReq(colesSplit));
|
||||
const manual = await pool.query(
|
||||
`SELECT t.id FROM transactions t JOIN expense_metadata em ON em.transaction_id = t.id
|
||||
WHERE em.order_reference = 'pantry:coles:7777:114:2153:2026-07-11#2'`
|
||||
);
|
||||
const statementId = await statementLine(73.82, "2026-07-12", "COLES 7777 WERRIBEE");
|
||||
await reconcileAs(manual.rows[0].id, statementId);
|
||||
|
||||
// Reconciled manual rows are excluded from figures by reconciled_with_id; what remains
|
||||
// live is the gift-card leg plus the statement line = the $114.57 that was actually spent.
|
||||
const { rows } = await pool.query(
|
||||
`SELECT coalesce(sum(amount), 0)::text AS total FROM transactions
|
||||
WHERE reconciled_with_id IS NULL AND superseded_by_id IS NULL`
|
||||
);
|
||||
expect(Number(rows[0].total)).toBeCloseTo(114.57, 2);
|
||||
});
|
||||
|
||||
it("leaves the gift-card leg alone", async () => {
|
||||
await ingest(ingestReq(colesSplit));
|
||||
const manual = await pool.query(
|
||||
`SELECT t.id FROM transactions t JOIN expense_metadata em ON em.transaction_id = t.id
|
||||
WHERE em.order_reference = 'pantry:coles:7777:114:2153:2026-07-11#2'`
|
||||
);
|
||||
const statementId = await statementLine(73.82, "2026-07-12", "COLES 7777 WERRIBEE");
|
||||
await reconcileAs(manual.rows[0].id, statementId);
|
||||
|
||||
const gift = await pool.query(
|
||||
`SELECT t.reconciled_with_id, t.payment_method FROM transactions t
|
||||
JOIN expense_metadata em ON em.transaction_id = t.id
|
||||
WHERE em.order_reference = 'pantry:coles:7777:114:2153:2026-07-11#1'`
|
||||
);
|
||||
expect(gift.rows[0].reconciled_with_id).toBeNull();
|
||||
expect(gift.rows[0].payment_method).toBe("credits");
|
||||
});
|
||||
|
||||
it("keeps the other source's metadata when the statement row already has some", async () => {
|
||||
// An emailed or Paperless copy of the same purchase may have got there first.
|
||||
// transaction_id is UNIQUE, so one of them has to lose — and it must lose visibly
|
||||
// rather than by constraint violation at 11pm.
|
||||
await ingest(ingestReq(colesSplit));
|
||||
const manual = await pool.query(
|
||||
`SELECT t.id FROM transactions t JOIN expense_metadata em ON em.transaction_id = t.id
|
||||
WHERE em.order_reference = 'pantry:coles:7777:114:2153:2026-07-11#2'`
|
||||
);
|
||||
const statementId = await statementLine(73.82, "2026-07-12", "COLES 7777 WERRIBEE");
|
||||
await pool.query(
|
||||
`INSERT INTO expense_metadata (transaction_id, source, order_reference, line_items)
|
||||
VALUES ($1, 'email', 'email:already-here', '[]'::jsonb)`,
|
||||
[statementId]
|
||||
);
|
||||
|
||||
const res = await reconcileAs(manual.rows[0].id, statementId);
|
||||
expect(res.status ?? 200).toBe(200);
|
||||
|
||||
const incumbent = await pool.query(`SELECT source FROM expense_metadata WHERE transaction_id = $1`, [statementId]);
|
||||
expect(incumbent.rows.map((r) => r.source)).toEqual(["email"]);
|
||||
|
||||
const pantryRow = await pool.query(`SELECT flags FROM expense_metadata WHERE order_reference = $1`, [
|
||||
"pantry:coles:7777:114:2153:2026-07-11#2",
|
||||
]);
|
||||
expect(JSON.stringify(pantryRow.rows[0].flags)).toContain("metadata_collision_on_reconcile");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { processReceiptIngestion, ReceiptValidationError, validateReceipt, type ParsedReceipt } from "@/lib/receipt-ingestion";
|
||||
|
||||
/**
|
||||
* Machine ingest endpoint for grocery receipts scanned in pantry-app.
|
||||
*
|
||||
* Sibling to /api/orders/ingest and deliberately shaped like it. Auth is a shared secret
|
||||
* rather than the Traefik `x-forwarded-user` header: this is called app-to-app, and there
|
||||
* is no browser session to forward.
|
||||
*
|
||||
* Its own token rather than ORDER_INGEST_TOKEN so pantry's credential can be rotated
|
||||
* without touching the n8n order flow, which runs on a schedule nobody is watching.
|
||||
*/
|
||||
function authorised(req: NextRequest): boolean {
|
||||
const expected = process.env.RECEIPT_INGEST_TOKEN;
|
||||
if (!expected) return false; // fail closed when unconfigured
|
||||
const got = req.headers.get("x-ingest-token");
|
||||
return !!got && got === expected;
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
if (!authorised(req)) return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
|
||||
let body: ParsedReceipt & { dryRun?: boolean };
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json({ error: "invalid JSON" }, { status: 400 });
|
||||
}
|
||||
|
||||
if (!body?.merchant_name || !body?.transaction_date || typeof body?.total !== "number" || !Number.isInteger(body?.capture_event_id)) {
|
||||
return NextResponse.json({ error: "merchant_name, transaction_date, total and capture_event_id are required" }, { status: 400 });
|
||||
}
|
||||
|
||||
try {
|
||||
// Dry run validates and reports what would be written without writing it — the same
|
||||
// affordance every maintenance script in pantry has, and for the same reason: the first
|
||||
// pass over a new receipt format is worth reading before it becomes money.
|
||||
if (body.dryRun) return NextResponse.json({ kind: "receipt", dryRun: true, flags: validateReceipt(body) });
|
||||
const result = await processReceiptIngestion(body);
|
||||
return NextResponse.json({ kind: "receipt", ...result });
|
||||
} catch (e) {
|
||||
// A receipt that will not validate is the failure that matters: it means the payment
|
||||
// side was read wrong, and booking it anyway would put a number in the ledger nobody
|
||||
// can stand behind. Loud, like OrderParseError.
|
||||
if (e instanceof ReceiptValidationError) {
|
||||
return NextResponse.json({ kind: "rejected", reason: e.message }, { status: 422 });
|
||||
}
|
||||
const message = e instanceof Error ? e.message : String(e);
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -77,6 +77,36 @@ export async function POST(req: NextRequest) {
|
||||
await tx.transaction_splits.deleteMany({ where: { transaction_id: manual_id } });
|
||||
}
|
||||
|
||||
// Move provenance: manual → statement tx.
|
||||
//
|
||||
// Overrides, tags and splits above were always carried across; expense_metadata was
|
||||
// the one child left behind, which did not matter while every metadata row came from
|
||||
// an email that had created its own transaction. It matters now: a scanned grocery
|
||||
// receipt puts its line items here, and reconciliation hides the manual row from
|
||||
// every figure — so without this the shop's contents disappear at exactly the moment
|
||||
// the statement line appears, and `COLES 0556 MANOR LAKES` stays as unreadable as it
|
||||
// was before the receipt was ever scanned.
|
||||
//
|
||||
// transaction_id is UNIQUE, so a statement row that already has metadata (an emailed
|
||||
// or Paperless copy got there first) keeps it. The pantry row stays attached to the
|
||||
// reconciled manual transaction and is flagged, rather than raising a constraint
|
||||
// violation or silently overwriting the other source.
|
||||
const moved = await tx.$executeRawUnsafe(
|
||||
`UPDATE expense_metadata SET transaction_id = $1
|
||||
WHERE transaction_id = $2
|
||||
AND NOT EXISTS (SELECT 1 FROM expense_metadata other WHERE other.transaction_id = $1)`,
|
||||
statement_tx_id,
|
||||
manual_id
|
||||
);
|
||||
if (moved === 0) {
|
||||
await tx.$executeRawUnsafe(
|
||||
`UPDATE expense_metadata
|
||||
SET flags = coalesce(flags, '[]'::jsonb) || '["metadata_collision_on_reconcile"]'::jsonb
|
||||
WHERE transaction_id = $1`,
|
||||
manual_id
|
||||
);
|
||||
}
|
||||
|
||||
// Mark manual tx as reconciled (link to statement tx)
|
||||
await tx.$executeRawUnsafe(
|
||||
`UPDATE transactions SET reconciled_with_id = $1 WHERE id = $2`,
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
import { queryRaw, queryRow } from "@/lib/db";
|
||||
import { DEFAULT_OWNER_ID } from "@/lib/order-ingestion";
|
||||
|
||||
/**
|
||||
* Grocery receipts scanned in pantry-app, arriving as candidate spend.
|
||||
*
|
||||
* The shape deliberately mirrors the order lane rather than inventing a second mechanism,
|
||||
* but it makes one decision differently and the difference is the point: **nothing is
|
||||
* parked.** An order can wait for its statement because it is already visible as an email;
|
||||
* a gift-card grocery shop is visible nowhere at all, so a scan that does not produce a
|
||||
* transaction produces nothing a person can see. Every leg becomes a manual transaction
|
||||
* (`statement_id IS NULL`) immediately, and finance's existing pending-reconciliation queue
|
||||
* resolves the ones that have a card leg coming.
|
||||
*
|
||||
* Why one transaction per tender leg rather than one per receipt: a shop settled $40.75 on
|
||||
* a gift card and $73.82 on a Mastercard has a statement line for $73.82 and nothing for
|
||||
* the rest. A single $114.57 row marked `credits` is excluded from the queue and lets the
|
||||
* statement line double-count; marked `card` it is searched for at ±1% of $114.57 and never
|
||||
* matches, so it parks forever while the statement line still counts. Either way the shop
|
||||
* books $188.39. Per-leg rows make each amount exactly the settled amount, so the existing
|
||||
* matcher works untouched.
|
||||
*/
|
||||
|
||||
export type TenderLeg = {
|
||||
leg_index: number;
|
||||
amount: number;
|
||||
card_last4?: string | null;
|
||||
card_product?: string | null;
|
||||
/** pantry's vocabulary; mapped to finance's payment_method below. */
|
||||
class?: "card" | "gift_card" | "cash" | null;
|
||||
};
|
||||
|
||||
export type ReceiptLineItem = {
|
||||
name: string;
|
||||
quantity?: number | null;
|
||||
unit?: string | null;
|
||||
line_total?: number | null;
|
||||
category?: string | null;
|
||||
};
|
||||
|
||||
export type ParsedReceipt = {
|
||||
receipt_uid?: string | null;
|
||||
capture_event_id: number;
|
||||
image_sha256?: string | null;
|
||||
merchant_name: string;
|
||||
store_detail?: string | null;
|
||||
transaction_date: string;
|
||||
total: number;
|
||||
tax_amount?: number | null;
|
||||
tender_raw?: string | null;
|
||||
loyalty_card_number?: string | null;
|
||||
tender_legs: TenderLeg[];
|
||||
line_items: ReceiptLineItem[];
|
||||
};
|
||||
|
||||
export type ReceiptIngestResult = {
|
||||
group: string;
|
||||
legs: { leg_index: number; transactionId: number; metadataId: number; paymentMethod: string | null; skipped?: string }[];
|
||||
flags: string[];
|
||||
};
|
||||
|
||||
export class ReceiptValidationError extends Error {}
|
||||
|
||||
/**
|
||||
* `credits` rather than a new `gift_card`: prepaid value with no card leg is a concept
|
||||
* finance already has, and reusing it inherits both the `needsCardMatch()` exclusion that
|
||||
* keeps such rows out of the reconciliation queue and the "Gift Card" label `bankLabel()`
|
||||
* renders for them. A second word for one idea would have needed both taught again.
|
||||
*
|
||||
* An unknown class maps to NULL, which `needsCardMatch()` treats as reconcilable — so a leg
|
||||
* nobody could classify lands in the queue visibly unresolved instead of being silently
|
||||
* decided either way.
|
||||
*/
|
||||
function paymentMethodFor(legClass: TenderLeg["class"]): string | null {
|
||||
if (legClass === "gift_card") return "credits";
|
||||
if (legClass === "cash") return "cash";
|
||||
if (legClass === "card") return "card";
|
||||
return null;
|
||||
}
|
||||
|
||||
const round2 = (value: number) => Number(value.toFixed(2));
|
||||
|
||||
/**
|
||||
* The whole shop is `groceries`. What it was actually made of — food versus household —
|
||||
* stays derived from `line_items` at display time rather than stored, because a mixed shop
|
||||
* is one payment and splitting the transaction to describe it would make the amount that
|
||||
* reconciles against the statement line stop matching it.
|
||||
*/
|
||||
const RECEIPT_CATEGORY = "groceries";
|
||||
|
||||
export function validateReceipt(receipt: ParsedReceipt): string[] {
|
||||
const flags: string[] = [];
|
||||
if (!receipt.tender_legs?.length) throw new ReceiptValidationError("at least one tender leg is required");
|
||||
if (!Number.isFinite(receipt.total) || receipt.total <= 0) throw new ReceiptValidationError("total must be a positive number");
|
||||
if (!/^\d{4}-\d{2}-\d{2}$/.test(receipt.transaction_date)) throw new ReceiptValidationError("transaction_date must be YYYY-MM-DD");
|
||||
|
||||
// The check that detects a split at all, and the one that proves the payment side was
|
||||
// read whole. A leg missed here becomes spend that never appears.
|
||||
const legSum = round2(receipt.tender_legs.reduce((total, leg) => total + Number(leg.amount || 0), 0));
|
||||
if (Math.abs(legSum - round2(receipt.total)) > 0.02) {
|
||||
throw new ReceiptValidationError(`tender legs sum to ${legSum.toFixed(2)} but the receipt total is ${receipt.total.toFixed(2)}`);
|
||||
}
|
||||
|
||||
// Lines are allowed to disagree: promotional rows are deliberately skipped during
|
||||
// extraction, so this flags rather than rejects. The money is the tender, not the lines.
|
||||
const lineSum = round2((receipt.line_items ?? []).reduce((total, line) => total + Number(line.line_total || 0), 0));
|
||||
if (receipt.line_items?.length && Math.abs(lineSum - round2(receipt.total)) > 0.02) flags.push(`line_items_sum_${lineSum.toFixed(2)}`);
|
||||
if (receipt.tender_legs.length > 1) flags.push("split_tender");
|
||||
if (receipt.tender_legs.some((leg) => !leg.class)) flags.push("unclassified_tender");
|
||||
return flags;
|
||||
}
|
||||
|
||||
/**
|
||||
* The receipt's own identity, not the capture's. A photo and the store's e-receipt PDF of
|
||||
* one purchase are different files with different hashes, so keying on the capture would
|
||||
* let the same shop arrive twice as two unrelated sets of transactions. Falls back to the
|
||||
* capture id when the receipt did not print enough to identify itself — that risks a
|
||||
* duplicate, which is visible and removable, rather than a merge, which silently hides a
|
||||
* real shop.
|
||||
*/
|
||||
export function receiptGroup(receipt: ParsedReceipt): string {
|
||||
return `pantry:${receipt.receipt_uid || `capture:${receipt.capture_event_id}`}`;
|
||||
}
|
||||
|
||||
export async function processReceiptIngestion(receipt: ParsedReceipt): Promise<ReceiptIngestResult> {
|
||||
const flags = validateReceipt(receipt);
|
||||
const group = receiptGroup(receipt);
|
||||
// Line items describe the whole shop but can only attach to one row —
|
||||
// expense_metadata.transaction_id is UNIQUE, and duplicating them would double any
|
||||
// composition derived from them. They go on the card leg because that is the row which
|
||||
// reconciles onto the statement line, which is where an unreadable `COLES 0556` descriptor
|
||||
// actually gets its contents. With no card leg, the largest leg carries them.
|
||||
const cardLeg = receipt.tender_legs.find((leg) => leg.class === "card" || !leg.class)
|
||||
?? [...receipt.tender_legs].sort((a, b) => Number(b.amount) - Number(a.amount))[0];
|
||||
|
||||
const legs: ReceiptIngestResult["legs"] = [];
|
||||
for (const leg of receipt.tender_legs) {
|
||||
const reference = `${group}#${leg.leg_index}`;
|
||||
const existing = await queryRow<{ id: number; transaction_id: number | null }>(
|
||||
`SELECT id, transaction_id FROM expense_metadata WHERE source = 'pantry' AND order_reference = $1`,
|
||||
[reference]
|
||||
);
|
||||
if (existing) {
|
||||
legs.push({ leg_index: leg.leg_index, transactionId: existing.transaction_id ?? 0, metadataId: existing.id, paymentMethod: null, skipped: "already_ingested" });
|
||||
continue;
|
||||
}
|
||||
|
||||
const paymentMethod = paymentMethodFor(leg.class);
|
||||
const amount = round2(Number(leg.amount));
|
||||
const description = receipt.store_detail ? `${receipt.merchant_name} ${receipt.store_detail}` : receipt.merchant_name;
|
||||
const txn = await queryRow<{ id: number }>(
|
||||
`INSERT INTO transactions (
|
||||
transaction_date, description, amount, amount_aud, category, payment_method,
|
||||
merchant_name, merchant_normalized, transaction_type, owner_id
|
||||
) VALUES ($1,$2,$3,$3,$4,$5,$6,$6,'debit',$7) RETURNING id`,
|
||||
[receipt.transaction_date, description, amount, RECEIPT_CATEGORY, paymentMethod, receipt.merchant_name, DEFAULT_OWNER_ID]
|
||||
);
|
||||
|
||||
const carriesLines = leg.leg_index === cardLeg?.leg_index;
|
||||
const meta = await queryRow<{ id: number }>(
|
||||
`INSERT INTO expense_metadata (
|
||||
transaction_id, source, order_reference, line_items, subtotal, amount,
|
||||
merchant_normalized, transaction_date, card_last4, currency, flags,
|
||||
reconciled_at, payment_method, payment_method_detail, tender_raw,
|
||||
receipt_sha256, receipt_group, extraction_model
|
||||
) VALUES ($1,'pantry',$2,$3::jsonb,$4,$5,$6,$7,$8,'AUD',$9::jsonb,NULL,$10,$11,$12,$13,$14,'cloud-budget')
|
||||
RETURNING id`,
|
||||
[
|
||||
txn!.id,
|
||||
reference,
|
||||
JSON.stringify(carriesLines ? receipt.line_items ?? [] : []),
|
||||
receipt.tax_amount ?? null,
|
||||
amount,
|
||||
receipt.merchant_name,
|
||||
receipt.transaction_date,
|
||||
leg.card_last4 ?? null,
|
||||
JSON.stringify(carriesLines ? flags : [...flags, "line_items_on_card_leg"]),
|
||||
paymentMethod,
|
||||
leg.card_product ?? null,
|
||||
receipt.tender_raw ?? null,
|
||||
receipt.image_sha256 ?? null,
|
||||
group,
|
||||
]
|
||||
);
|
||||
legs.push({ leg_index: leg.leg_index, transactionId: txn!.id, metadataId: meta!.id, paymentMethod });
|
||||
}
|
||||
return { group, legs, flags };
|
||||
}
|
||||
|
||||
/**
|
||||
* Statement lines that look like a pantry row already marked as needing no card leg.
|
||||
*
|
||||
* Such rows never enter the reconciliation queue — that is the whole point of classifying
|
||||
* them — so a mis-learned card would otherwise double-count in silence. This only flags: the
|
||||
* same uncertainty that makes the classification fallible makes the match a suggestion, and
|
||||
* auto-reconciling on it would trade a visible error for an invisible one. A confirmed
|
||||
* conflict is a reason to correct the card's stored class, which fixes every later receipt
|
||||
* from it at once.
|
||||
*/
|
||||
export async function pantryTenderConflicts(): Promise<{ transactionId: number; statementTransactionId: number; amount: string; merchant: string | null; date: string }[]> {
|
||||
return queryRaw(
|
||||
`SELECT p.id AS "transactionId", s.id AS "statementTransactionId", p.amount::text AS amount,
|
||||
p.merchant_normalized AS merchant, p.transaction_date::text AS date
|
||||
FROM transactions p
|
||||
JOIN expense_metadata em ON em.transaction_id = p.id AND em.source = 'pantry'
|
||||
JOIN transactions s ON s.statement_id IS NOT NULL
|
||||
AND s.transaction_date BETWEEN p.transaction_date - 3 AND p.transaction_date + 3
|
||||
AND s.amount BETWEEN p.amount * 0.99 AND p.amount * 1.01
|
||||
AND upper(coalesce(s.description, '')) LIKE '%' || upper(p.merchant_name) || '%'
|
||||
WHERE p.statement_id IS NULL
|
||||
AND p.reconciled_with_id IS NULL
|
||||
AND p.payment_method IN ('cash', 'credits')
|
||||
ORDER BY p.transaction_date DESC`
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user