siddharthd/finance-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(orders): wire the real parser in, defer card reconciliation

Browse Source 
Ingestion now runs on the rebuilt parser. Three substantive changes.

Deferred card reconciliation. A 'MasterCard 8032 and/or credits' receipt never
states the split, but the card leg lands on the statement — Subway's $29.08
order shows $13.06 on 8032, so $16.02 was credits. For a live order that
statement is weeks away, so the split cannot be settled at ingest time. Such
orders are now parked with provenance and no transaction, and
reconcilePendingOrders() resolves them once the statement arrives. Backfill
takes the same path and resolves immediately. Migration 0019 adds the columns
that make an order resumable; applied to personal_test only, prod untouched.

Payment detection bug, found by the new tests: the old regex delimited the
'Paid with' line on a double space, which whitespace collapsing removes. Every
card and mixed receipt fell through to the credits branch — the Woolworths
receipt booked $60.93 of credits spend that never happened.

Category resolution reversed deliberately. Correction 1 said never default to
dining; the implementation of that sent everything unrecognised to 'other', and
knowing six merchants meant Carl's Jr, Taco Bell, Chilli India, Oporto, Schnitz
and Souvlaki GR all landed there. Grocers are an enumerable set and restaurants
are not, so match groceries explicitly and let the residual be dining.

Tests rebuilt on real captured receipts; the synthetic fixtures are deleted.
60 unit + 41 integration green on three consecutive runs.
This commit is contained in:
siddharthd
2026-07-26 22:43:10 +10:00
parent 33db7d05ef
commit c82a22767f
10 changed files with 626 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/__tests__/unit/order-ingestion.test.ts
+112 -53
View File
@@ -1,77 +1,136 @@
import { describe, it, expect } from "vitest";
import { readFileSync } from "fs";
import { resolve } from "path";
import { parseOrderHTML, validateOrderTotals, resolveMerchantCategory } from "../../lib/order-ingestion";
import {
parseOrderHTML,
validateOrderTotals,
resolveCategory,
OrderParseError,
type MessageMeta,
type ParsedOrder,
} from "../../lib/order-ingestion";
describe("Order Ingestion - Unit Tests", () => {
const fixturesDir = resolve(__dirname, "../fixtures/orders");
/**
* Rewritten 2026-07-26. The previous unit suite exercised synthetic fixtures
* built to satisfy the parser, so it passed while the parser could not read a
* real email. These run against unmodified captured receipts.
*/
const dir = resolve(__dirname, "../fixtures/orders/real");
const html = (f: string) => readFileSync(resolve(dir, `${f}.html`), "utf-8");
const meta = (over: Partial<MessageMeta> = {}): MessageMeta => ({
messageId: "unit-1",
subject: "Order Confirmation for Siddharth from Mad Mex",
receivedAt: "2026-07-16T03:34:00Z",
sender: "DoorDash Order <no-reply@doordash.com>",
...over,
});
it("1. total_charged extracted, not subtotal — Mad Mex fixture => 14.64 (I3)", () => {
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
const parsed = parseOrderHTML(html);
expect(parsed.totals.total_charged).toBe(14.64);
expect(parsed.totals.subtotal).toBe(22.10);
describe("payment detection", () => {
it("credits-only", () => {
const p = parseOrderHTML(html("dd-01"), meta());
expect(p.payment.credits_amount).toBe(14.64);
expect(p.payment.card_last4).toBeNull();
expect(p.payment.ambiguous).toBe(false);
});
it("2. Discount from structured HTML => 9.45, not 24.09 (I9)", () => {
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
const parsed = parseOrderHTML(html);
expect(parsed.totals.discounts).toBe(9.45);
expect(parsed.totals.discounts).not.toBe(24.09);
it("card-only produces no credits figure", () => {
const p = parseOrderHTML(
html("dd-27"),
meta({ subject: "Order Confirmation for Siddharth from Subway" })
);
expect(p.payment.card_last4).toBe("8032");
expect(p.payment.credits_amount).toBeNull();
});
it("3. Arithmetic validation rejects a tampered fixture (totals ±$1)", () => {
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
const parsed = parseOrderHTML(html);
parsed.totals.total_charged = 100.00; // Tampered
const isValid = validateOrderTotals(parsed);
expect(isValid).toBe(false);
it("'and/or credits' is ambiguous, not silently credits", () => {
// Regression: an earlier regex delimited the payment line on a double
// space, which whitespace collapsing removes. Every card and mixed receipt
// fell through to the credits branch — this one booked the whole $60.93 as
// credits spend that never happened.
const p = parseOrderHTML(
html("dd-10"),
meta({ subject: "Order Confirmation for Siddharth from Woolworths" })
);
expect(p.payment.ambiguous).toBe(true);
expect(p.payment.card_last4).toBe("8032");
expect(p.payment.credits_amount).toBeNull();
});
});
it("4. Missing payment line => nulls, not a guess", () => {
const html = "<div>Total Charged $15.00</div><div>Subtotal $15.00</div>";
const parsed = parseOrderHTML(html);
expect(parsed.payment.credits_amount).toBeNull();
expect(parsed.payment.card_amount).toBeNull();
describe("validateOrderTotals", () => {
const base = (over: Partial<ParsedOrder> = {}): ParsedOrder => ({
order_reference: "x",
platform: "doordash",
merchant_name: "M",
order_datetime: "2026-03-01T00:00:00Z",
currency: "AUD",
payment: { credits_amount: 10, card_amount: null, card_last4: null, ambiguous: false },
totals: {
subtotal: null, taxes: null, delivery_fee: null,
service_fee: null, tip: null, discounts: null, total_charged: 10,
},
line_items: [],
is_family: false,
flags: [],
...over,
});
it("5. Grocery fixture => groceries; assert category != 'dining' (I4)", () => {
const html = readFileSync(resolve(fixturesDir, "doordash-credits-grocery.html"), "utf-8");
const parsed = parseOrderHTML(html);
const resolved = resolveMerchantCategory(parsed.merchant_name);
expect(resolved.category).toBe("groceries");
expect(resolved.category).not.toBe("dining");
it("rejects a non-positive total", () => {
const o = base();
o.totals.total_charged = 0;
expect(validateOrderTotals(o).ok).toBe(false);
});
it("6. Unresolvable merchant => other + review flag, never dining", () => {
const resolved = resolveMerchantCategory("Random Food Truck");
expect(resolved.category).toBe("other");
expect(resolved.flagReview).toBe(true);
expect(resolved.category).not.toBe("dining");
it("rejects payments that do not account for the total", () => {
const r = validateOrderTotals(
base({ payment: { credits_amount: 5, card_amount: null, card_last4: null, ambiguous: false } })
);
expect(r.ok).toBe(false);
expect(r.reason).toMatch(/payments sum/);
});
it("7. Rating scale accepts only 'again', 'fine', 'never'", () => {
const validRatings = ["again", "fine", "never"];
const invalidRating = "superb";
expect(validRatings.includes("again")).toBe(true);
expect(validRatings.includes(invalidRating)).toBe(false);
it("accepts a matching total", () => {
expect(validateOrderTotals(base()).ok).toBe(true);
});
it("8. Uber Eats receipt => platform 'ubereats', line_items [] (Diff 1), Uber Cash mapped", () => {
const html = `<div>Your Friday evening order with Uber Eats</div><div>Paid with Uber Cash $43.46</div><div>Total Charged $43.46</div>`;
const parsed = parseOrderHTML(html);
expect(parsed.platform).toBe("ubereats");
expect(parsed.payment.credits_amount).toBe(43.46);
expect(parsed.line_items).toEqual([]);
it("does not gate on DoorDash's non-reconciling fee breakdown", () => {
const p = parseOrderHTML(html("dd-01"), meta());
expect(p.totals.subtotal).toBe(22.10);
expect(p.totals.discounts).toBe(24.09); // 22.10 + 1.99 — genuinely printed
expect(validateOrderTotals(p, html("dd-01")).ok).toBe(true);
});
});
it("9. Uber Ride receipt => platform 'uber', category 'transport' (Diff 2)", () => {
const html = `<div>Your Trip fare $84.14 · Booking Fee $1.35</div><div>Uber Cash $85.49</div><div>Total $85.49</div>`;
const parsed = parseOrderHTML(html);
expect(parsed.platform).toBe("uber");
expect(parsed.payment.credits_amount).toBe(85.49);
describe("resolveCategory", () => {
const o = (merchant: string, platform: ParsedOrder["platform"] = "doordash") =>
({ merchant_name: merchant, platform }) as ParsedOrder;
const resolved = resolveMerchantCategory(parsed.merchant_name, parsed.platform);
expect(resolved.category).toBe("transport");
it("maps grocers to groceries", () => {
expect(resolveCategory(o("Woolworths"))).toBe("groceries");
expect(resolveCategory(o("ALDI"))).toBe("groceries");
expect(resolveCategory(o("GLOMARK Kandana", "ubereats"))).toBe("groceries");
});
it("maps restaurants to dining rather than 'other'", () => {
// The earlier six-merchant allowlist sent every one of these to `other`.
for (const m of ["Carl's Jr.", "Taco Bell", "Chilli India", "Oporto", "Schnitz", "Souvlaki GR"]) {
expect(resolveCategory(o(m))).toBe("dining");
}
});
it("maps rides to transport", () => {
expect(resolveCategory(o("Uber Trip", "uber"))).toBe("transport");
});
});
describe("parse guards", () => {
it("throws on a body too short to be a receipt", () => {
expect(() => parseOrderHTML("<html></html>", meta())).toThrow(OrderParseError);
});
it("throws rather than inventing a platform", () => {
expect(() =>
parseOrderHTML(html("dd-01"), meta({ subject: "Newsletter", sender: "someone@example.com" }))
).toThrow(/platform/i);
});
});