diff --git a/prisma/migrations/0027_pantry_receipt_source/migration.sql b/prisma/migrations/0027_pantry_receipt_source/migration.sql new file mode 100644 index 0000000..3aff1c6 --- /dev/null +++ b/prisma/migrations/0027_pantry_receipt_source/migration.sql @@ -0,0 +1,37 @@ +-- Receipt scans as a second producer on the order lane. +-- +-- A grocery shop paid with a supermarket gift card settles against no statement and +-- arrives in no mail, so the pantry scan is the only touchpoint that ever sees it. Rather +-- than a second ingestion mechanism, a scan lands as a manual transaction +-- (statement_id IS NULL) and the existing pending-reconciliation queue resolves it — +-- with needsCardMatch() already excluding cash/credits rows for which no card leg is +-- ever coming. + +-- The receipt as text, kept as evidence rather than parsed into a decision. The token +-- that distinguishes a gift card from a bank card was only findable by reading two +-- payments side by side on one receipt; whether it holds across merchants is answerable +-- from stored blocks and not at all from none. +ALTER TABLE expense_metadata + ADD COLUMN IF NOT EXISTS tender_raw TEXT; + +-- The image the extraction came from. Not the idempotency key -- a photo and the store's +-- e-receipt PDF of one purchase hash differently -- but exact where it applies, and the +-- only thing a later cross-source dedupe against an emailed or Paperless copy could match +-- on. +ALTER TABLE expense_metadata + ADD COLUMN IF NOT EXISTS receipt_sha256 TEXT; + +CREATE INDEX IF NOT EXISTS idx_expense_receipt_sha256 + ON expense_metadata (receipt_sha256) + WHERE receipt_sha256 IS NOT NULL; + +-- Legs of one split-tender shop, so a $40.75 gift-card row can be shown as part of a +-- $114.57 purchase instead of an orphan. The shared receipt identity is carried in +-- order_reference ('pantry:#'); this column is +-- what makes the group queryable without parsing that string. +ALTER TABLE expense_metadata + ADD COLUMN IF NOT EXISTS receipt_group TEXT; + +CREATE INDEX IF NOT EXISTS idx_expense_receipt_group + ON expense_metadata (receipt_group) + WHERE receipt_group IS NOT NULL; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index dde4ba6..77f1c7d 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -215,6 +215,20 @@ model expense_metadata { transaction_date DateTime? @db.Date extraction_model String? @default("gemini-2.5-flash") created_at DateTime? @default(now()) + // In the database since migrations 0019/0020 but absent from this model until 0027. + // Regenerating the client from the stale definition would have dropped columns the order + // lane writes on every ingest. + card_last4 String? + currency String? + flags Json @default("[]") + reconciled_at DateTime? @db.Timestamptz(6) + matched_transaction_id Int? + platform String? + route Json? + // 0027 — receipt scans as a second producer on this lane. + tender_raw String? + receipt_sha256 String? + receipt_group String? transaction transactions? @relation(fields: [transaction_id], references: [id], onDelete: Cascade) @@unique([source, order_reference], name: "uq_expense_source_order") diff --git a/src/__tests__/integration/receipt-ingest.test.ts b/src/__tests__/integration/receipt-ingest.test.ts new file mode 100644 index 0000000..f30c757 --- /dev/null +++ b/src/__tests__/integration/receipt-ingest.test.ts @@ -0,0 +1,231 @@ +import { describe, it, expect, beforeAll, beforeEach } from "vitest"; +import { Pool } from "pg"; +import { createPool, resetDB, seedParticipants } from "./helpers"; + +/** + * The receipt lane, exercised against the real schema. + * + * Every fixture here is a receipt that actually exists: a Coles split-tender e-receipt + * (7777 Werribee, 11/07/2026, $114.57 settled $40.75 gift card + $73.82 Mastercard), a + * Woolworths e-receipt (3345 Wyndham Vale, 29/06/2026, $40.75, single gift card) and a + * Coles photo (556 Manor Lakes, 29/07/2026, $23.18). The split one is the reason the lane + * writes one transaction per tender leg rather than one per receipt, so it is the case + * these tests are built around. + */ + +const TOKEN = "test-receipt-token"; +let pool: Pool; +let POST: typeof import("../../app/api/receipts/ingest/route").POST; + +const req = (body: unknown, token: string | null = TOKEN) => + ({ + headers: { get: (h: string) => (h === "x-ingest-token" ? token : null) }, + json: async () => body, + }) as never; + +const colesSplit = () => ({ + receipt_uid: "coles:7777:114:2153:2026-07-11", + capture_event_id: 412, + image_sha256: "sha-coles-split", + merchant_name: "Coles", + store_detail: "7777", + transaction_date: "2026-07-11", + total: 114.57, + tender_raw: "EFT $40.75\nEFT $73.82\n***** 0443 MASTERCARD\nCREDIT ACCOUNT STORE CARD", + tender_legs: [ + { leg_index: 1, amount: 40.75, card_last4: "0443", card_product: "STORE CARD", class: "gift_card" as const }, + { leg_index: 2, amount: 73.82, card_last4: "3302", card_product: "MASTERCARD", class: "card" as const }, + ], + line_items: [ + { name: "Pork Loin Roast", quantity: 1, unit: "ea", line_total: 15.86, category: "meat" }, + { name: "Tomatoes", quantity: 1, unit: "ea", line_total: 98.71, category: "produce" }, + ], +}); + +const woolworthsGiftCard = () => ({ + receipt_uid: "woolworths:3345:62:148:2026-06-29", + capture_event_id: 500, + merchant_name: "Woolworths", + store_detail: "3345", + transaction_date: "2026-06-29", + total: 40.75, + tender_legs: [{ leg_index: 1, amount: 40.75, card_last4: "0443", card_product: "STORE CARD", class: "gift_card" as const }], + line_items: [{ name: "Oat Milk", quantity: 2, unit: "ea", line_total: 40.75, category: "dairy" }], +}); + +beforeAll(async () => { + process.env.RECEIPT_INGEST_TOKEN = TOKEN; + pool = createPool(); + ({ POST } = await import("../../app/api/receipts/ingest/route")); +}); + +beforeEach(async () => { + await resetDB(pool); + await pool.query("DELETE FROM expense_metadata"); + // transactions.owner_id references participants, and resetDB truncates it with RESTART + // IDENTITY — so DEFAULT_OWNER_ID (1) has to be re-seeded or every insert here fails the + // foreign key. Same dependency the order lane has; it just never had to say so. + await seedParticipants(pool); +}); + +const legsOf = async (group: string) => + (await pool.query( + `SELECT t.id, t.amount::text, t.payment_method, t.statement_id, em.order_reference, em.line_items, em.flags + FROM expense_metadata em JOIN transactions t ON t.id = em.transaction_id + WHERE em.receipt_group = $1 ORDER BY em.order_reference`, + [group] + )).rows; + +describe("auth", () => { + it("rejects a missing or wrong token", async () => { + expect((await POST(req(colesSplit(), null))).status).toBe(401); + expect((await POST(req(colesSplit(), "nope"))).status).toBe(401); + }); + + it("rejects a body missing what it needs to book money", async () => { + expect((await POST(req({ merchant_name: "Coles" }))).status).toBe(400); + }); +}); + +describe("a split-tender receipt", () => { + it("books one transaction per leg, not one for the total", async () => { + const res = await POST(req(colesSplit())); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.legs).toHaveLength(2); + + const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11"); + expect(rows.map((r) => r.amount)).toEqual(["40.75", "73.82"]); + // Not one $114.57 row: marked credits it would let the $73.82 statement line + // double-count; marked card it would be searched for at ±1% of $114.57 and never match. + expect(rows.some((r) => r.amount === "114.57")).toBe(false); + }); + + it("sums the legs back to the printed total", async () => { + await POST(req(colesSplit())); + const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11"); + const total = rows.reduce((sum, r) => sum + Number(r.amount), 0); + expect(Number(total.toFixed(2))).toBe(114.57); + }); + + it("marks the gift-card leg credits and the card leg card", async () => { + await POST(req(colesSplit())); + const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11"); + expect(rows.find((r) => r.amount === "40.75")!.payment_method).toBe("credits"); + expect(rows.find((r) => r.amount === "73.82")!.payment_method).toBe("card"); + }); + + it("leaves every leg as a manual row for the reconciliation queue", async () => { + await POST(req(colesSplit())); + const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11"); + expect(rows.every((r) => r.statement_id === null)).toBe(true); + }); + + it("puts the line items on the card leg only", async () => { + // They describe the whole shop but transaction_id is UNIQUE on expense_metadata, so + // they can attach to one row — and the card leg is the one that reconciles onto the + // statement line, which is where an unreadable descriptor gets its contents. + await POST(req(colesSplit())); + const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11"); + expect(rows.find((r) => r.amount === "73.82")!.line_items).toHaveLength(2); + expect(rows.find((r) => r.amount === "40.75")!.line_items).toHaveLength(0); + }); + + it("flags the split so it can be shown as one purchase", async () => { + await POST(req(colesSplit())); + const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11"); + expect(rows[0].flags).toContain("split_tender"); + }); +}); + +describe("a gift-card receipt", () => { + it("becomes a visible transaction on the day it is scanned", async () => { + const res = await POST(req(woolworthsGiftCard())); + expect(res.status).toBe(200); + const rows = await legsOf("pantry:woolworths:3345:62:148:2026-06-29"); + expect(rows).toHaveLength(1); + expect(rows[0].amount).toBe("40.75"); + expect(rows[0].payment_method).toBe("credits"); + }); + + it("is owned, so it is not invisible in every view", async () => { + // Owner scoping is COALESCE(t.owner_id, s.owner_id) and these rows carry no statement; + // a NULL owner is how 85 backfilled order rows ended up in the table and on no screen. + await POST(req(woolworthsGiftCard())); + const { rows } = await pool.query("SELECT owner_id FROM transactions WHERE merchant_name = 'Woolworths'"); + expect(rows[0].owner_id).not.toBeNull(); + }); +}); + +describe("idempotency", () => { + it("keys on the receipt, so the same shop from two files lands once", async () => { + // A photo and the store's e-receipt PDF are different files with different hashes. + // Keying on the capture would let one purchase arrive twice. + await POST(req(colesSplit())); + await POST(req({ ...colesSplit(), capture_event_id: 999, image_sha256: "sha-different-file" })); + const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11"); + expect(rows).toHaveLength(2); // still just the two legs + }); + + it("reports the replay rather than silently doing nothing", async () => { + await POST(req(colesSplit())); + const body = await (await POST(req(colesSplit()))).json(); + expect(body.legs.every((l: { skipped?: string }) => l.skipped === "already_ingested")).toBe(true); + }); + + it("falls back to the capture when the receipt did not identify itself", async () => { + // A crumpled photo can lose the header entirely. That receipt still becomes spend; it + // just cannot be recognised if the same shop is scanned again from another file. + await POST(req({ ...colesSplit(), receipt_uid: null })); + expect(await legsOf("pantry:capture:412")).toHaveLength(2); + }); +}); + +describe("validation — what becomes money", () => { + it("refuses a receipt whose tender does not add up to its total", async () => { + // A missed leg is spend that never appears; booking the rest would put a number in the + // ledger nobody can stand behind. + const broken = { ...colesSplit(), tender_legs: [colesSplit().tender_legs[0]] }; + const res = await POST(req(broken)); + expect(res.status).toBe(422); + expect((await res.json()).reason).toContain("tender legs sum to 40.75"); + expect(await legsOf("pantry:coles:7777:114:2153:2026-07-11")).toHaveLength(0); + }); + + it("refuses a receipt with no tender at all", async () => { + expect((await POST(req({ ...colesSplit(), tender_legs: [] }))).status).toBe(422); + }); + + it("flags but accepts lines that do not sum, because promo rows are skipped by design", async () => { + const res = await POST(req({ ...colesSplit(), line_items: [{ name: "One line", line_total: 10 }] })); + expect(res.status).toBe(200); + expect((await res.json()).flags.some((f: string) => f.startsWith("line_items_sum_"))).toBe(true); + }); + + it("writes nothing at all when validation fails", async () => { + await POST(req({ ...colesSplit(), tender_legs: [] })); + const { rows } = await pool.query("SELECT count(*)::int AS n FROM transactions"); + expect(rows[0].n).toBe(0); + }); + + it("marks a leg nobody could classify as reconcilable rather than deciding for them", async () => { + const unknown = { ...colesSplit(), tender_legs: [{ leg_index: 1, amount: 114.57, card_last4: "9999", card_product: null, class: null }] }; + const res = await POST(req(unknown)); + expect(res.status).toBe(200); + const rows = await legsOf("pantry:coles:7777:114:2153:2026-07-11"); + // NULL is what needsCardMatch() treats as still needing a card match, so it surfaces in + // the queue instead of being silently excluded from it. + expect(rows[0].payment_method).toBeNull(); + expect(rows[0].flags).toContain("unclassified_tender"); + }); +}); + +describe("dry run", () => { + it("validates without writing", async () => { + const res = await POST(req({ ...colesSplit(), dryRun: true })); + expect(res.status).toBe(200); + expect((await res.json()).dryRun).toBe(true); + const { rows } = await pool.query("SELECT count(*)::int AS n FROM transactions"); + expect(rows[0].n).toBe(0); + }); +}); diff --git a/src/__tests__/integration/receipt-reconcile.test.ts b/src/__tests__/integration/receipt-reconcile.test.ts new file mode 100644 index 0000000..7827b6d --- /dev/null +++ b/src/__tests__/integration/receipt-reconcile.test.ts @@ -0,0 +1,166 @@ +import { describe, it, expect, beforeAll, beforeEach, vi } from "vitest"; +import { Pool } from "pg"; +import { createPool, resetDB, seedParticipants } from "./helpers"; + +/** + * What happens to a scanned receipt when its statement finally arrives. + * + * This is the case the first design of this lane got wrong, so it is tested first-class. + * Reconciliation moves a manual row's overrides, tags and splits onto the statement row and + * then hides the manual row from every figure. `expense_metadata` was the one child it left + * behind — which did not matter while metadata only ever came from an email that had made + * its own transaction, and matters completely now that it carries a shop's line items. Left + * unmoved, the contents of the shop vanish at exactly the moment the statement line shows + * up, and `COLES 0556 MANOR LAKES` stays as unreadable as it was before anything was + * scanned. + */ + +const TOKEN = "test-receipt-token"; +let pool: Pool; +let ownerId: number; +let ingest: typeof import("../../app/api/receipts/ingest/route").POST; +let reconcile: typeof import("../../app/api/transactions/reconcile/route").POST; + +const ingestReq = (body: unknown) => + ({ headers: { get: (h: string) => (h === "x-ingest-token" ? TOKEN : null) }, json: async () => body }) as never; + +// The reconcile route authenticates a browser session rather than a shared secret. +const userReq = (body: unknown) => ({ json: async () => body }) as never; + +const colesSplit = { + receipt_uid: "coles:7777:114:2153:2026-07-11", + capture_event_id: 412, + merchant_name: "Coles", + transaction_date: "2026-07-11", + total: 114.57, + tender_legs: [ + { leg_index: 1, amount: 40.75, card_last4: "0443", card_product: "STORE CARD", class: "gift_card" }, + { leg_index: 2, amount: 73.82, card_last4: "3302", card_product: "MASTERCARD", class: "card" }, + ], + line_items: [ + { name: "Pork Loin Roast", quantity: 1, unit: "ea", line_total: 15.86, category: "meat" }, + { name: "Jasmine Rice 1kg", quantity: 1, unit: "ea", line_total: 98.71, category: "pantry_dry" }, + ], +}; + +async function statementLine(amount: number, date: string, description: string) { + const statement = await pool.query( + `INSERT INTO statements (filename, account_number, bank_name, billing_start_date, billing_end_date, owner_id) + VALUES ('sept.pdf', '1234-5678-9012-3302', 'Test Bank', $1::date - 20, $1::date + 10, $2) RETURNING id`, + [date, ownerId] + ); + const txn = await pool.query( + `INSERT INTO transactions (statement_id, transaction_date, description, amount, transaction_type, owner_id) + VALUES ($1, $2, $3, $4, 'debit', $5) RETURNING id`, + [statement.rows[0].id, date, description, amount, ownerId] + ); + return txn.rows[0].id as number; +} + +beforeAll(async () => { + process.env.RECEIPT_INGEST_TOKEN = TOKEN; + pool = createPool(); + // The reconcile route scopes every query to the signed-in user. Mocked before the import + // (doMock is not hoisted, so it can close over `ownerId`, which resetDB reassigns on each + // run) because an ES module binding cannot be reassigned afterwards. + vi.doMock("@/lib/auth", () => ({ + getCurrentUser: async () => ({ id: ownerId, name: "Alice", email: "alice@example.com" }), + })); + ({ POST: ingest } = await import("../../app/api/receipts/ingest/route")); + ({ POST: reconcile } = await import("../../app/api/transactions/reconcile/route")); +}); + +beforeEach(async () => { + await resetDB(pool); + await pool.query("DELETE FROM expense_metadata"); + ({ ownerId } = await seedParticipants(pool)); +}); + +const reconcileAs = (manualId: number, statementId: number) => + reconcile(userReq({ matches: [{ manual_id: manualId, statement_tx_id: statementId }] })); + +describe("a card leg meeting its statement line", () => { + it("carries the shop's line items onto the statement row", async () => { + await ingest(ingestReq(colesSplit)); + const manual = await pool.query( + `SELECT t.id FROM transactions t JOIN expense_metadata em ON em.transaction_id = t.id + WHERE em.order_reference = 'pantry:coles:7777:114:2153:2026-07-11#2'` + ); + const manualId = manual.rows[0].id as number; + const statementId = await statementLine(73.82, "2026-07-12", "COLES 7777 WERRIBEE"); + + const res = await reconcileAs(manualId, statementId); + expect(res.status ?? 200).toBe(200); + + const moved = await pool.query(`SELECT transaction_id, line_items FROM expense_metadata WHERE order_reference = $1`, [ + "pantry:coles:7777:114:2153:2026-07-11#2", + ]); + // The whole point: the items are now on the row that survives, not the one that got hidden. + expect(moved.rows[0].transaction_id).toBe(statementId); + expect(moved.rows[0].line_items).toHaveLength(2); + }); + + it("counts the shop once, not twice", async () => { + await ingest(ingestReq(colesSplit)); + const manual = await pool.query( + `SELECT t.id FROM transactions t JOIN expense_metadata em ON em.transaction_id = t.id + WHERE em.order_reference = 'pantry:coles:7777:114:2153:2026-07-11#2'` + ); + const statementId = await statementLine(73.82, "2026-07-12", "COLES 7777 WERRIBEE"); + await reconcileAs(manual.rows[0].id, statementId); + + // Reconciled manual rows are excluded from figures by reconciled_with_id; what remains + // live is the gift-card leg plus the statement line = the $114.57 that was actually spent. + const { rows } = await pool.query( + `SELECT coalesce(sum(amount), 0)::text AS total FROM transactions + WHERE reconciled_with_id IS NULL AND superseded_by_id IS NULL` + ); + expect(Number(rows[0].total)).toBeCloseTo(114.57, 2); + }); + + it("leaves the gift-card leg alone", async () => { + await ingest(ingestReq(colesSplit)); + const manual = await pool.query( + `SELECT t.id FROM transactions t JOIN expense_metadata em ON em.transaction_id = t.id + WHERE em.order_reference = 'pantry:coles:7777:114:2153:2026-07-11#2'` + ); + const statementId = await statementLine(73.82, "2026-07-12", "COLES 7777 WERRIBEE"); + await reconcileAs(manual.rows[0].id, statementId); + + const gift = await pool.query( + `SELECT t.reconciled_with_id, t.payment_method FROM transactions t + JOIN expense_metadata em ON em.transaction_id = t.id + WHERE em.order_reference = 'pantry:coles:7777:114:2153:2026-07-11#1'` + ); + expect(gift.rows[0].reconciled_with_id).toBeNull(); + expect(gift.rows[0].payment_method).toBe("credits"); + }); + + it("keeps the other source's metadata when the statement row already has some", async () => { + // An emailed or Paperless copy of the same purchase may have got there first. + // transaction_id is UNIQUE, so one of them has to lose — and it must lose visibly + // rather than by constraint violation at 11pm. + await ingest(ingestReq(colesSplit)); + const manual = await pool.query( + `SELECT t.id FROM transactions t JOIN expense_metadata em ON em.transaction_id = t.id + WHERE em.order_reference = 'pantry:coles:7777:114:2153:2026-07-11#2'` + ); + const statementId = await statementLine(73.82, "2026-07-12", "COLES 7777 WERRIBEE"); + await pool.query( + `INSERT INTO expense_metadata (transaction_id, source, order_reference, line_items) + VALUES ($1, 'email', 'email:already-here', '[]'::jsonb)`, + [statementId] + ); + + const res = await reconcileAs(manual.rows[0].id, statementId); + expect(res.status ?? 200).toBe(200); + + const incumbent = await pool.query(`SELECT source FROM expense_metadata WHERE transaction_id = $1`, [statementId]); + expect(incumbent.rows.map((r) => r.source)).toEqual(["email"]); + + const pantryRow = await pool.query(`SELECT flags FROM expense_metadata WHERE order_reference = $1`, [ + "pantry:coles:7777:114:2153:2026-07-11#2", + ]); + expect(JSON.stringify(pantryRow.rows[0].flags)).toContain("metadata_collision_on_reconcile"); + }); +}); diff --git a/src/app/api/receipts/ingest/route.ts b/src/app/api/receipts/ingest/route.ts new file mode 100644 index 0000000..c32bd06 --- /dev/null +++ b/src/app/api/receipts/ingest/route.ts @@ -0,0 +1,52 @@ +import { NextRequest, NextResponse } from "next/server"; +import { processReceiptIngestion, ReceiptValidationError, validateReceipt, type ParsedReceipt } from "@/lib/receipt-ingestion"; + +/** + * Machine ingest endpoint for grocery receipts scanned in pantry-app. + * + * Sibling to /api/orders/ingest and deliberately shaped like it. Auth is a shared secret + * rather than the Traefik `x-forwarded-user` header: this is called app-to-app, and there + * is no browser session to forward. + * + * Its own token rather than ORDER_INGEST_TOKEN so pantry's credential can be rotated + * without touching the n8n order flow, which runs on a schedule nobody is watching. + */ +function authorised(req: NextRequest): boolean { + const expected = process.env.RECEIPT_INGEST_TOKEN; + if (!expected) return false; // fail closed when unconfigured + const got = req.headers.get("x-ingest-token"); + return !!got && got === expected; +} + +export async function POST(req: NextRequest) { + if (!authorised(req)) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + + let body: ParsedReceipt & { dryRun?: boolean }; + try { + body = await req.json(); + } catch { + return NextResponse.json({ error: "invalid JSON" }, { status: 400 }); + } + + if (!body?.merchant_name || !body?.transaction_date || typeof body?.total !== "number" || !Number.isInteger(body?.capture_event_id)) { + return NextResponse.json({ error: "merchant_name, transaction_date, total and capture_event_id are required" }, { status: 400 }); + } + + try { + // Dry run validates and reports what would be written without writing it — the same + // affordance every maintenance script in pantry has, and for the same reason: the first + // pass over a new receipt format is worth reading before it becomes money. + if (body.dryRun) return NextResponse.json({ kind: "receipt", dryRun: true, flags: validateReceipt(body) }); + const result = await processReceiptIngestion(body); + return NextResponse.json({ kind: "receipt", ...result }); + } catch (e) { + // A receipt that will not validate is the failure that matters: it means the payment + // side was read wrong, and booking it anyway would put a number in the ledger nobody + // can stand behind. Loud, like OrderParseError. + if (e instanceof ReceiptValidationError) { + return NextResponse.json({ kind: "rejected", reason: e.message }, { status: 422 }); + } + const message = e instanceof Error ? e.message : String(e); + return NextResponse.json({ error: message }, { status: 500 }); + } +} diff --git a/src/app/api/transactions/reconcile/route.ts b/src/app/api/transactions/reconcile/route.ts index e920ae1..0763b38 100644 --- a/src/app/api/transactions/reconcile/route.ts +++ b/src/app/api/transactions/reconcile/route.ts @@ -77,6 +77,36 @@ export async function POST(req: NextRequest) { await tx.transaction_splits.deleteMany({ where: { transaction_id: manual_id } }); } + // Move provenance: manual → statement tx. + // + // Overrides, tags and splits above were always carried across; expense_metadata was + // the one child left behind, which did not matter while every metadata row came from + // an email that had created its own transaction. It matters now: a scanned grocery + // receipt puts its line items here, and reconciliation hides the manual row from + // every figure — so without this the shop's contents disappear at exactly the moment + // the statement line appears, and `COLES 0556 MANOR LAKES` stays as unreadable as it + // was before the receipt was ever scanned. + // + // transaction_id is UNIQUE, so a statement row that already has metadata (an emailed + // or Paperless copy got there first) keeps it. The pantry row stays attached to the + // reconciled manual transaction and is flagged, rather than raising a constraint + // violation or silently overwriting the other source. + const moved = await tx.$executeRawUnsafe( + `UPDATE expense_metadata SET transaction_id = $1 + WHERE transaction_id = $2 + AND NOT EXISTS (SELECT 1 FROM expense_metadata other WHERE other.transaction_id = $1)`, + statement_tx_id, + manual_id + ); + if (moved === 0) { + await tx.$executeRawUnsafe( + `UPDATE expense_metadata + SET flags = coalesce(flags, '[]'::jsonb) || '["metadata_collision_on_reconcile"]'::jsonb + WHERE transaction_id = $1`, + manual_id + ); + } + // Mark manual tx as reconciled (link to statement tx) await tx.$executeRawUnsafe( `UPDATE transactions SET reconciled_with_id = $1 WHERE id = $2`, diff --git a/src/lib/receipt-ingestion.ts b/src/lib/receipt-ingestion.ts new file mode 100644 index 0000000..52e6864 --- /dev/null +++ b/src/lib/receipt-ingestion.ts @@ -0,0 +1,215 @@ +import { queryRaw, queryRow } from "@/lib/db"; +import { DEFAULT_OWNER_ID } from "@/lib/order-ingestion"; + +/** + * Grocery receipts scanned in pantry-app, arriving as candidate spend. + * + * The shape deliberately mirrors the order lane rather than inventing a second mechanism, + * but it makes one decision differently and the difference is the point: **nothing is + * parked.** An order can wait for its statement because it is already visible as an email; + * a gift-card grocery shop is visible nowhere at all, so a scan that does not produce a + * transaction produces nothing a person can see. Every leg becomes a manual transaction + * (`statement_id IS NULL`) immediately, and finance's existing pending-reconciliation queue + * resolves the ones that have a card leg coming. + * + * Why one transaction per tender leg rather than one per receipt: a shop settled $40.75 on + * a gift card and $73.82 on a Mastercard has a statement line for $73.82 and nothing for + * the rest. A single $114.57 row marked `credits` is excluded from the queue and lets the + * statement line double-count; marked `card` it is searched for at ±1% of $114.57 and never + * matches, so it parks forever while the statement line still counts. Either way the shop + * books $188.39. Per-leg rows make each amount exactly the settled amount, so the existing + * matcher works untouched. + */ + +export type TenderLeg = { + leg_index: number; + amount: number; + card_last4?: string | null; + card_product?: string | null; + /** pantry's vocabulary; mapped to finance's payment_method below. */ + class?: "card" | "gift_card" | "cash" | null; +}; + +export type ReceiptLineItem = { + name: string; + quantity?: number | null; + unit?: string | null; + line_total?: number | null; + category?: string | null; +}; + +export type ParsedReceipt = { + receipt_uid?: string | null; + capture_event_id: number; + image_sha256?: string | null; + merchant_name: string; + store_detail?: string | null; + transaction_date: string; + total: number; + tax_amount?: number | null; + tender_raw?: string | null; + loyalty_card_number?: string | null; + tender_legs: TenderLeg[]; + line_items: ReceiptLineItem[]; +}; + +export type ReceiptIngestResult = { + group: string; + legs: { leg_index: number; transactionId: number; metadataId: number; paymentMethod: string | null; skipped?: string }[]; + flags: string[]; +}; + +export class ReceiptValidationError extends Error {} + +/** + * `credits` rather than a new `gift_card`: prepaid value with no card leg is a concept + * finance already has, and reusing it inherits both the `needsCardMatch()` exclusion that + * keeps such rows out of the reconciliation queue and the "Gift Card" label `bankLabel()` + * renders for them. A second word for one idea would have needed both taught again. + * + * An unknown class maps to NULL, which `needsCardMatch()` treats as reconcilable — so a leg + * nobody could classify lands in the queue visibly unresolved instead of being silently + * decided either way. + */ +function paymentMethodFor(legClass: TenderLeg["class"]): string | null { + if (legClass === "gift_card") return "credits"; + if (legClass === "cash") return "cash"; + if (legClass === "card") return "card"; + return null; +} + +const round2 = (value: number) => Number(value.toFixed(2)); + +/** + * The whole shop is `groceries`. What it was actually made of — food versus household — + * stays derived from `line_items` at display time rather than stored, because a mixed shop + * is one payment and splitting the transaction to describe it would make the amount that + * reconciles against the statement line stop matching it. + */ +const RECEIPT_CATEGORY = "groceries"; + +export function validateReceipt(receipt: ParsedReceipt): string[] { + const flags: string[] = []; + if (!receipt.tender_legs?.length) throw new ReceiptValidationError("at least one tender leg is required"); + if (!Number.isFinite(receipt.total) || receipt.total <= 0) throw new ReceiptValidationError("total must be a positive number"); + if (!/^\d{4}-\d{2}-\d{2}$/.test(receipt.transaction_date)) throw new ReceiptValidationError("transaction_date must be YYYY-MM-DD"); + + // The check that detects a split at all, and the one that proves the payment side was + // read whole. A leg missed here becomes spend that never appears. + const legSum = round2(receipt.tender_legs.reduce((total, leg) => total + Number(leg.amount || 0), 0)); + if (Math.abs(legSum - round2(receipt.total)) > 0.02) { + throw new ReceiptValidationError(`tender legs sum to ${legSum.toFixed(2)} but the receipt total is ${receipt.total.toFixed(2)}`); + } + + // Lines are allowed to disagree: promotional rows are deliberately skipped during + // extraction, so this flags rather than rejects. The money is the tender, not the lines. + const lineSum = round2((receipt.line_items ?? []).reduce((total, line) => total + Number(line.line_total || 0), 0)); + if (receipt.line_items?.length && Math.abs(lineSum - round2(receipt.total)) > 0.02) flags.push(`line_items_sum_${lineSum.toFixed(2)}`); + if (receipt.tender_legs.length > 1) flags.push("split_tender"); + if (receipt.tender_legs.some((leg) => !leg.class)) flags.push("unclassified_tender"); + return flags; +} + +/** + * The receipt's own identity, not the capture's. A photo and the store's e-receipt PDF of + * one purchase are different files with different hashes, so keying on the capture would + * let the same shop arrive twice as two unrelated sets of transactions. Falls back to the + * capture id when the receipt did not print enough to identify itself — that risks a + * duplicate, which is visible and removable, rather than a merge, which silently hides a + * real shop. + */ +export function receiptGroup(receipt: ParsedReceipt): string { + return `pantry:${receipt.receipt_uid || `capture:${receipt.capture_event_id}`}`; +} + +export async function processReceiptIngestion(receipt: ParsedReceipt): Promise { + const flags = validateReceipt(receipt); + const group = receiptGroup(receipt); + // Line items describe the whole shop but can only attach to one row — + // expense_metadata.transaction_id is UNIQUE, and duplicating them would double any + // composition derived from them. They go on the card leg because that is the row which + // reconciles onto the statement line, which is where an unreadable `COLES 0556` descriptor + // actually gets its contents. With no card leg, the largest leg carries them. + const cardLeg = receipt.tender_legs.find((leg) => leg.class === "card" || !leg.class) + ?? [...receipt.tender_legs].sort((a, b) => Number(b.amount) - Number(a.amount))[0]; + + const legs: ReceiptIngestResult["legs"] = []; + for (const leg of receipt.tender_legs) { + const reference = `${group}#${leg.leg_index}`; + const existing = await queryRow<{ id: number; transaction_id: number | null }>( + `SELECT id, transaction_id FROM expense_metadata WHERE source = 'pantry' AND order_reference = $1`, + [reference] + ); + if (existing) { + legs.push({ leg_index: leg.leg_index, transactionId: existing.transaction_id ?? 0, metadataId: existing.id, paymentMethod: null, skipped: "already_ingested" }); + continue; + } + + const paymentMethod = paymentMethodFor(leg.class); + const amount = round2(Number(leg.amount)); + const description = receipt.store_detail ? `${receipt.merchant_name} ${receipt.store_detail}` : receipt.merchant_name; + const txn = await queryRow<{ id: number }>( + `INSERT INTO transactions ( + transaction_date, description, amount, amount_aud, category, payment_method, + merchant_name, merchant_normalized, transaction_type, owner_id + ) VALUES ($1,$2,$3,$3,$4,$5,$6,$6,'debit',$7) RETURNING id`, + [receipt.transaction_date, description, amount, RECEIPT_CATEGORY, paymentMethod, receipt.merchant_name, DEFAULT_OWNER_ID] + ); + + const carriesLines = leg.leg_index === cardLeg?.leg_index; + const meta = await queryRow<{ id: number }>( + `INSERT INTO expense_metadata ( + transaction_id, source, order_reference, line_items, subtotal, amount, + merchant_normalized, transaction_date, card_last4, currency, flags, + reconciled_at, payment_method, payment_method_detail, tender_raw, + receipt_sha256, receipt_group, extraction_model + ) VALUES ($1,'pantry',$2,$3::jsonb,$4,$5,$6,$7,$8,'AUD',$9::jsonb,NULL,$10,$11,$12,$13,$14,'cloud-budget') + RETURNING id`, + [ + txn!.id, + reference, + JSON.stringify(carriesLines ? receipt.line_items ?? [] : []), + receipt.tax_amount ?? null, + amount, + receipt.merchant_name, + receipt.transaction_date, + leg.card_last4 ?? null, + JSON.stringify(carriesLines ? flags : [...flags, "line_items_on_card_leg"]), + paymentMethod, + leg.card_product ?? null, + receipt.tender_raw ?? null, + receipt.image_sha256 ?? null, + group, + ] + ); + legs.push({ leg_index: leg.leg_index, transactionId: txn!.id, metadataId: meta!.id, paymentMethod }); + } + return { group, legs, flags }; +} + +/** + * Statement lines that look like a pantry row already marked as needing no card leg. + * + * Such rows never enter the reconciliation queue — that is the whole point of classifying + * them — so a mis-learned card would otherwise double-count in silence. This only flags: the + * same uncertainty that makes the classification fallible makes the match a suggestion, and + * auto-reconciling on it would trade a visible error for an invisible one. A confirmed + * conflict is a reason to correct the card's stored class, which fixes every later receipt + * from it at once. + */ +export async function pantryTenderConflicts(): Promise<{ transactionId: number; statementTransactionId: number; amount: string; merchant: string | null; date: string }[]> { + return queryRaw( + `SELECT p.id AS "transactionId", s.id AS "statementTransactionId", p.amount::text AS amount, + p.merchant_normalized AS merchant, p.transaction_date::text AS date + FROM transactions p + JOIN expense_metadata em ON em.transaction_id = p.id AND em.source = 'pantry' + JOIN transactions s ON s.statement_id IS NOT NULL + AND s.transaction_date BETWEEN p.transaction_date - 3 AND p.transaction_date + 3 + AND s.amount BETWEEN p.amount * 0.99 AND p.amount * 1.01 + AND upper(coalesce(s.description, '')) LIKE '%' || upper(p.merchant_name) || '%' + WHERE p.statement_id IS NULL + AND p.reconciled_with_id IS NULL + AND p.payment_method IN ('cash', 'credits') + ORDER BY p.transaction_date DESC` + ); +}