From 0823244e4a114275acd6e1958b784f3a5dfa1764 Mon Sep 17 00:00:00 2001 From: siddharthd Date: Sat, 22 Aug 2026 11:46:05 +1000 Subject: [PATCH] intake: the word "Image" is not a receipt, and the prompt must not be data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Batch 3d2ffbe8 arrived as one text part whose entire content was the five characters `Image` — an iOS Shortcut coerced the shared photo to text. The model was asked anyway and answered out of its prompt: Burgertory (Braybrook), two "The Pride" lines at $22.80, a $41.15 total. Nothing on it existed. Burgertory was in ORDER_ITEM_PROMPT as the example of how to print a branch in parentheses. Only the missing date kept it out of the matcher; the invented header and items were stored on the batch regardless. Three layers, none of them a model: 1. textCouldBeReceipt() — deterministic, before any model call. A TEXT part with fewer than two words, no digit, or under 40 characters and no money figure cannot be a receipt. Length is NOT the test on its own: `TOTAL $17.00` is twelve characters and is exactly what this door accepts. A rejected part is dropped like an unreadable file type, so one junk part beside a good screenshot still goes through; if nothing usable is left the card quotes what the phone actually sent. PDFs and images untouched. 2. The prompt names no real shop — merchant examples are PROMPT_PLACEHOLDER_MERCHANTS, exported from intake-rules so the prompt and the guard cannot drift — and it is told what to return when the input is not a receipt at all. 3. confabulationReason() — the belt. Reachable where the pre-check cannot look: a PDF that renders to a scrap of junk still reaches the model. Also rejects any answer whose merchant IS a placeholder. All three file `unsupported`, never `needs_header`. A confabulated header is not a partial one, and the needs_header path stores what it is given. Same shape as the ingestion engine's 2026-08-05 empty-body defect: a stage that can be handed nothing has to say so, because the model downstream will always answer. Tests: 16 unit cases over the two pure functions and the prompt; three integration cases — `Image` is declined with zero model calls and nothing but `unsupported` on the batch row, a junk part beside a real receipt still enriches, and a junk PDF is unsupported rather than needs_header. The first and third fail on the tree before this commit. --- CLAUDE.md | 19 ++++ src/__tests__/integration/intake.test.ts | 58 ++++++++++++ src/__tests__/unit/intake-rules.test.ts | 107 ++++++++++++++++++++++- src/lib/intake-extract.ts | 13 ++- src/lib/intake-router.ts | 52 ++++++++++- src/lib/intake-rules.ts | 91 +++++++++++++++++++ 6 files changed, 335 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 06d1419..5d42039 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -699,6 +699,25 @@ and the cross-repo contract are in smarthome `plans/intake-router-slice-plan.md` delivery order's total is items + fees − promotions ($118.94 of Coles lines became $85.74), so checking lines against the total would hold every delivery order. Tolerance $1 or 2%; a failure is `held`, nothing written. +- **Nothing that cannot be a receipt reaches the model, and the prompt names no + real shop.** On 2026-08-22 an iOS Shortcut coerced a shared photo to text, so + a batch arrived as the five characters `Image` — and the model returned a + whole receipt: `Burgertory (Braybrook)`, two "The Pride" lines, $41.15. None + of it existed; "Burgertory" was in `ORDER_ITEM_PROMPT` as the example of how + to print a branch in parentheses. Only the absent date kept it out of the + matcher, and the invented header and items were still stored on the batch as + `needs_header`. Three layers now, in order: `textCouldBeReceipt()` rejects a + TEXT part with fewer than two words, no digit, or under 40 characters with no + money figure (`TOTAL $17.00` passes — length alone is *not* the test) + **before** any model call; the prompt's merchant examples are + `PROMPT_PLACEHOLDER_MERCHANTS` format placeholders and it is told to return + nulls for a non-receipt; and `confabulationReason()` catches what the + pre-check cannot see — a PDF that renders to junk — plus any answer echoing a + placeholder. All three file `unsupported`, never `needs_header`: a + confabulated header is not a partial one. Same shape as the ingestion + engine's 2026-08-05 empty-body defect — **a stage that can be handed nothing + must say so, because the model downstream always answers, and it answers out + of its prompt.** - **Names stay verbatim.** Burgertory calls its burgers "Michael" and "Joel"; the prompt forbids canonicalising and the stored shape is the order lane's (`qty`, `description`, `amount`, `options`) so `order-details.tsx` renders it. diff --git a/src/__tests__/integration/intake.test.ts b/src/__tests__/integration/intake.test.ts index dc2e95a..53f36ab 100644 --- a/src/__tests__/integration/intake.test.ts +++ b/src/__tests__/integration/intake.test.ts @@ -339,6 +339,64 @@ describe("POST /api/intake — gates", () => { expect((await batch(id)).status).toBe("unsupported"); expect(calls).toHaveLength(0); }); + + // 2026-08-22, live batch 3d2ffbe8…: an iOS Shortcut coerced the shared photo + // to text, so the whole upload was the five characters `Image`. The model was + // asked anyway and returned Burgertory (Braybrook), two "The Pride" lines and + // a $41.15 total — none of which existed. Only the absent date kept it out of + // the matcher, and the invented header was stored on the batch as + // `needs_header`. The model must never be asked. + it("a text part that cannot be a receipt is declined BEFORE any model call", async () => { + const id = uuid(); + modelReply = { ...burgertory, merchant: "Burgertory (Braybrook)", order_date: null, total: 41.15 }; + const res = await intakePOST( + multipart({ source: "other", batch_id: id }, [{ name: "shared.txt", type: "text/plain", bytes: Buffer.from("Image") }]) + ); + const body = await res.json(); + expect(body).toMatchObject({ kind: "unsupported", status: "unsupported" }); + expect(body.text).toMatch(/single word/); + expect(body.text).toMatch(/"Image"/); + // The model was never asked — the only thing that makes confabulation impossible. + expect(calls).toHaveLength(0); + const row = await batch(id); + expect(row.status).toBe("unsupported"); + expect(row.kind).toBe("unsupported"); + expect(row.header).toBeNull(); + expect(row.items).toBeNull(); + expect(row.transaction_id).toBeNull(); + expect((await pool.query(`SELECT count(*)::int AS n FROM transactions`)).rows[0].n).toBe(0); + }); + + it("a junk text part beside a real receipt does not sink the batch", async () => { + const txn = await seedOrder({ merchant: "Burgertory (Werribee)", date: "2026-08-19", amount: 22.4 }); + const body = await ( + await intakePOST( + multipart({ source: "uber_eats" }, [ + { name: "shared.txt", type: "text/plain", bytes: Buffer.from("Image") }, + { name: "Receipt_19Aug2026.pdf", type: "application/pdf", bytes: BURGERTORY }, + ]) + ) + ).json(); + expect(body.status).toBe("enriched"); + expect(await lineItemsOf(txn)).toHaveLength(2); + }); + + it("an order extracted from an unreadable PDF is unsupported, not needs_header", async () => { + const id = uuid(); + // A PDF that renders to a scrap of junk still reaches the model — the + // pre-check only inspects text parts. `confabulationReason` is the belt. + vi.spyOn(await import("../../lib/intake-extract"), "extractPdfText").mockResolvedValueOnce("Image"); + modelReply = { ...burgertory, merchant: "Burgertory (Braybrook)" }; + const body = await ( + await intakePOST(multipart({ source: "uber_eats", batch_id: id }, [{ name: "x.pdf", type: "application/pdf", bytes: BURGERTORY }])) + ).json(); + expect(body).toMatchObject({ kind: "unsupported", status: "unsupported" }); + expect(body.text).toMatch(/nothing shared could be read/); + const row = await batch(id); + expect(row.status).toBe("unsupported"); + expect(row.header).toBeNull(); + expect(row.items).toBeNull(); + }); }); describe("POST /api/intake — food (enrich vs candidates)", () => { diff --git a/src/__tests__/unit/intake-rules.test.ts b/src/__tests__/unit/intake-rules.test.ts index ee9cd27..f5eac4f 100644 --- a/src/__tests__/unit/intake-rules.test.ts +++ b/src/__tests__/unit/intake-rules.test.ts @@ -3,17 +3,21 @@ import { readFileSync } from "fs"; import { resolve } from "path"; import { classifyInput, + confabulationReason, GROCERY_PATTERN, merchantKey, merchantMatches, platformForSource, + PROMPT_PLACEHOLDER_MERCHANTS, + quoteShared, reconcileItems, resolveMerchantKind, + textCouldBeReceipt, toLineItems, validateHeader, type ExtractedOrder, } from "../../lib/intake-rules"; -import { buildMessageContent, extractPdfText, parseExtraction, EXTRACT_RESPONSE_FORMAT, ExtractionError, liteLLMExtractor } from "../../lib/intake-extract"; +import { buildMessageContent, extractPdfText, parseExtraction, EXTRACT_RESPONSE_FORMAT, ExtractionError, liteLLMExtractor, ORDER_ITEM_PROMPT } from "../../lib/intake-extract"; import { askCard, candidatesCard, enrichedCard, type Block } from "../../lib/intake-blocks"; /** @@ -298,3 +302,104 @@ describe("liteLLMExtractor — a finish_reason that is present must be `stop`", await expect(extractor("tool_calls")([{ kind: "text", text: "x", name: "x" }])).rejects.toThrow(/tool_calls/); }); }); + +describe("textCouldBeReceipt — nothing that cannot be a receipt reaches the model", () => { + const why = (t: string | null | undefined) => { + const r = textCouldBeReceipt(t); + return r.ok ? null : r.reason; + }; + + it("the live defect: the phone sent the WORD `Image`, not the image", () => { + expect(textCouldBeReceipt("Image").ok).toBe(false); + expect(why("Image")).toMatch(/single word/); + expect(why("Image\n")).toMatch(/single word/); + }); + + it("a filename is not a receipt, digits in it notwithstanding", () => { + expect(why("IMG_0042.jpeg")).toMatch(/single word/); + expect(why("Receipt_19Aug2026.pdf")).toMatch(/single word/); + }); + + it("empty, blank and whitespace-only", () => { + expect(why("")).toMatch(/empty/); + expect(why(" \n\t ")).toMatch(/empty/); + expect(why(null)).toMatch(/empty/); + expect(why(undefined)).toMatch(/empty/); + }); + + it("prose with no numbers cannot be a receipt", () => { + expect(why("Shared from Photos")).toMatch(/no numbers/); + expect(why("here is the receipt for dinner last night, sorry it took so long")).toMatch(/no numbers/); + }); + + it("short but with a printed amount goes through — this door accepts a pasted line", () => { + expect(textCouldBeReceipt("TOTAL $17.00").ok).toBe(true); + expect(textCouldBeReceipt("Total 17.00").ok).toBe(true); + expect(textCouldBeReceipt("Gesamt 12,50 EUR").ok).toBe(true); + }); + + it("short with digits but no money figure does not — a screenshot name is not a purchase", () => { + expect(why("Screenshot 2026 08 22")).toMatch(/too short/); + expect(why("2 items")).toMatch(/too short/); + }); + + it("a real receipt body passes", () => { + expect( + textCouldBeReceipt("Burgertory (Werribee)\n19 Aug 2026\n1x Michael 14.00\n1x Joel 14.00\nSubtotal 28.00\nTotal 22.40").ok + ).toBe(true); + }); + + it("quoteShared flattens and truncates what goes on the card", () => { + expect(quoteShared("Image")).toBe("Image"); + expect(quoteShared(" a \n b ")).toBe("a b"); + expect(quoteShared("x".repeat(200))).toHaveLength(80); + expect(quoteShared("x".repeat(200)).endsWith("…")).toBe(true); + }); +}); + +describe("confabulationReason — the belt to that pre-check", () => { + const nothing = { textParts: ["Image"], hasImages: false }; + const readable = { textParts: ["Total $22.40 Burgertory (Werribee) 2026-08-19"], hasImages: false }; + const order = { merchant: "Burgertory (Braybrook)", items: [{ description: "The Pride" }] }; + + it("flags an order extracted from nothing readable", () => { + expect(confabulationReason(nothing, order)).toMatch(/nothing shared could be read/); + }); + + it("does not flag one extracted from something readable", () => { + expect(confabulationReason(readable, order)).toBeNull(); + }); + + it("does not flag an image share — the pixels are what we asked for", () => { + expect(confabulationReason({ textParts: [], hasImages: true }, order)).toBeNull(); + }); + + it("an empty answer to an unreadable share is honest, not a confabulation", () => { + expect(confabulationReason(nothing, { merchant: null, items: [] })).toBeNull(); + }); + + it("the prompt's own placeholder coming back as a merchant is always a confabulation", () => { + for (const p of PROMPT_PLACEHOLDER_MERCHANTS) { + expect(confabulationReason(readable, { merchant: p, items: [] })).toMatch(/prompt's own example/); + expect(confabulationReason({ textParts: [], hasImages: true }, { merchant: p, items: [] })).toMatch(/prompt's own example/); + } + }); + + it("a real merchant that happens to be named in the prompt's platform list is NOT flagged", () => { + expect(confabulationReason(readable, { merchant: "Coles (Wyndham Vale)", items: [] })).toBeNull(); + expect(confabulationReason(readable, { merchant: "Woolworths", items: [] })).toBeNull(); + }); +}); + +describe("the prompt carries no real merchant to echo", () => { + it("names no shop it could hand back as data", () => { + expect(ORDER_ITEM_PROMPT).not.toMatch(/Burgertory/); + expect(ORDER_ITEM_PROMPT).not.toMatch(/Wyndham Vale/); + expect(ORDER_ITEM_PROMPT).toContain(PROMPT_PLACEHOLDER_MERCHANTS[0]); + }); + + it("tells the model what to return when the input is not a receipt", () => { + expect(ORDER_ITEM_PROMPT).toMatch(/NOT a receipt/); + expect(ORDER_ITEM_PROMPT).toMatch(/Do not construct a plausible order/); + }); +}); diff --git a/src/lib/intake-extract.ts b/src/lib/intake-extract.ts index 1195a00..5bb3895 100644 --- a/src/lib/intake-extract.ts +++ b/src/lib/intake-extract.ts @@ -1,5 +1,5 @@ import { extractText, getDocumentProxy } from "unpdf"; -import type { ExtractedOrder, ExtractedItem } from "@/lib/intake-rules"; +import { PROMPT_PLACEHOLDER_MERCHANTS, type ExtractedOrder, type ExtractedItem } from "@/lib/intake-rules"; /** * Extraction for the intake router: one joint model call per batch. @@ -31,13 +31,22 @@ export async function extractPdfText(bytes: Buffer): Promise { return (Array.isArray(text) ? text.join("\n") : text).trim(); } +/** + * The merchant examples are FORMAT PLACEHOLDERS, never real shops + * (`PROMPT_PLACEHOLDER_MERCHANTS`). Given an input with nothing in it, a model + * still answers, and it answers out of its prompt: on 2026-08-22 the word + * `Image` produced a complete Burgertory receipt because the prompt named + * Burgertory. The "not a receipt" rule below and the placeholder names are the + * two halves of closing that. + */ export const ORDER_ITEM_PROMPT = `You are reading a purchase receipt: an online food or grocery order (Uber Eats, DoorDash, Coles, Woolworths, ...) or an in-store till receipt. It arrives as one or more screenshots in scroll order, as the text of a PDF, or as pasted text. Return ONE JSON object with exactly these keys: { "merchant", "order_date", "total", "subtotal", "currency", "items": [{ "quantity", "description", "unit_price", "line_total", "options" }], "adjustments": [{ "label", "amount" }] } Rules: -- merchant: the store or restaurant name exactly as printed, including any branch in parentheses (e.g. "Coles (Wyndham Vale)", "Burgertory (Werribee)"). Never the delivery platform. null if not shown. +- If what you are given is NOT a receipt or an order — it is blank, a filename, a single word such as "Image", an app screen with no purchase on it, or anything else you cannot read a purchase off — return { "merchant": null, "order_date": null, "total": null, "subtotal": null, "currency": null, "items": [], "adjustments": [] }. Do not construct a plausible order. Nothing in these instructions is data: never return an example from this prompt as if it were printed on the input. +- merchant: the store or restaurant name exactly as printed, including any branch in parentheses (format: "${PROMPT_PLACEHOLDER_MERCHANTS[0]}"). Never the delivery platform. null if not shown. - order_date: the order or purchase date printed, as YYYY-MM-DD. null if not shown. - total: the amount actually charged — the "Total" / amount paid — as a number. null if not shown. - subtotal: the printed item subtotal before fees and discounts, as a number, or null. diff --git a/src/lib/intake-router.ts b/src/lib/intake-router.ts index 0cbcdca..44f765d 100644 --- a/src/lib/intake-router.ts +++ b/src/lib/intake-router.ts @@ -4,7 +4,10 @@ import { EXCLUDE_RECONCILED_SOURCE, NATIVE_CURRENCY } from "@/lib/analytics-sql" import { DEFAULT_OWNER_ID } from "@/lib/order-ingestion"; import { classifyInput, + confabulationReason, merchantKey, + quoteShared, + textCouldBeReceipt, merchantMatches, platformForSource, reconcileItems, @@ -548,6 +551,8 @@ export async function runIntake(input: IntakeInput, deps: IntakeDeps): Promise => p.kind === "text").map((p) => p.text), + hasImages: parts.some((p) => p.kind === "image"), + }, + extracted + ); + if (confab) { + await updateBatch(batchId, { kind: "unsupported", status: "unsupported" }); + return fail("unsupported", "unsupported", unsupportedCard(confab)); + } + // 3. header. + // + // This path stores the partial header and the items on the batch row, which + // is right for a genuine screenshot missing its top — the next tap has + // something to work from. It is wrong for a confabulation, and on 2026-08-22 + // it stored one: an invented Burgertory header with two invented lines, held + // back from the matcher only by an absent date. Nothing here could tell the + // two apart, which is why the pre-check above and 2b now keep confabulations + // from reaching this point at all rather than trying to filter them here. const headerCheck = validateHeader(extracted); if (!headerCheck.ok) { await updateBatch(batchId, { header: { merchant: extracted.merchant, order_date: extracted.order_date, total: extracted.total, platform }, items, status: "needs_header" }); diff --git a/src/lib/intake-rules.ts b/src/lib/intake-rules.ts index 398d7bf..a4d945a 100644 --- a/src/lib/intake-rules.ts +++ b/src/lib/intake-rules.ts @@ -250,3 +250,94 @@ export function merchantMatches(stored: string | null | undefined, extracted: st if (!a.length || !b.length) return false; return containsWordRun(a, b) || containsWordRun(b, a); } + +// ─── is this text even a receipt? (2026-08-22) ─────────────────────────────── + +/** + * The merchant names the extraction prompt uses to show the FORMAT. They are + * placeholders rather than real shops on purpose. + * + * On 2026-08-22 an iOS Shortcut coerced a shared photo to text, so the intake + * router received one text part whose entire content was the five characters + * `Image`. The model was handed that, and returned a whole receipt: + * `Burgertory (Braybrook)`, two "The Pride" lines at $22.80, a $41.15 total. + * Nothing on the receipt existed — "Burgertory" was in the prompt, as the + * example of how to print a branch in parentheses. Only the absent date kept it + * out of the matcher; the invented header and items were stored on the batch. + * + * Same shape as the ingestion engine's 2026-08-05 empty-body defect: a stage + * that can be handed nothing has to say so, because the model downstream will + * always answer, and it answers out of its prompt. + * + * Exported from here rather than written into the prompt string so the prompt + * and the guard that rejects an echo of it cannot drift apart. + */ +export const PROMPT_PLACEHOLDER_MERCHANTS = [" ()"] as const; + +/** + * Below this a text part needs a money figure to be believable. NOT a floor on + * its own: `TOTAL $17.00` is twelve characters and is exactly the kind of + * pasted line this door exists to accept. + */ +export const MIN_RECEIPT_TEXT_CHARS = 40; + +/** `$17.00`, `17.00`, `12,50` — a printed amount, not merely a digit. */ +const MONEY_SHAPE = /(?:[$€£¥]\s*\d|\d[.,]\d{2}(?!\d))/; + +export type TextCheck = { ok: true } | { ok: false; reason: string }; + +/** + * Can this text possibly be a receipt? Deterministic, before any model call. + * + * A receipt has more than one word and at least one number. Everything the + * phone substitutes when a share goes wrong — the literal word `Image`, a bare + * filename like `IMG_0042.jpeg`, an empty body — fails one of those, and none + * of them fails in a way a model will admit to. + * + * Applies to TEXT parts only. A PDF is checked for readable text separately and + * an image is the thing we actually asked for. + */ +export function textCouldBeReceipt(raw: string | null | undefined): TextCheck { + const text = (raw ?? "").trim(); + if (!text) return { ok: false, reason: "it was empty" }; + if (!/\s/.test(text)) return { ok: false, reason: "it is a single word" }; + if (!/\d/.test(text)) return { ok: false, reason: "it has no numbers in it" }; + if (text.length < MIN_RECEIPT_TEXT_CHARS && !MONEY_SHAPE.test(text)) { + return { ok: false, reason: "it is too short to be a receipt" }; + } + return { ok: true }; +} + +/** What the phone actually sent, short enough to put on a card. */ +export function quoteShared(raw: string, max = 80): string { + const flat = (raw ?? "").replace(/\s+/g, " ").trim(); + return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat; +} + +/** + * Belt to `textCouldBeReceipt`'s braces: did the extractor answer out of its + * prompt rather than out of the upload? + * + * Two tells, both cheap and both deterministic: + * - the merchant IS one of the prompt's format placeholders; or + * - nothing readable was sent at all (no images, and no text part that could + * be a receipt) and yet a merchant or items came back. + * + * The second is reachable even with the pre-check in place, because the + * pre-check only inspects text parts: a PDF that extracts to a few characters + * of junk still reaches the model. This is `unsupported`, never `needs_header` + * — a confabulated header is not a partial one. + */ +export function confabulationReason( + input: { textParts: string[]; hasImages: boolean }, + extracted: { merchant: string | null; items: unknown[] } +): string | null { + const key = merchantKey(extracted.merchant); + if (key && PROMPT_PLACEHOLDER_MERCHANTS.some((p) => merchantKey(p) === key)) { + return "the merchant came back as the prompt's own example rather than a name off the receipt"; + } + if (input.hasImages) return null; + if (input.textParts.some((t) => textCouldBeReceipt(t).ok)) return null; + if (!extracted.merchant && !extracted.items.length) return null; + return "nothing shared could be read as a receipt, yet an order came back"; +}