feat(orders): wire the real parser in, defer card reconciliation
Ingestion now runs on the rebuilt parser. Three substantive changes. Deferred card reconciliation. A 'MasterCard 8032 and/or credits' receipt never states the split, but the card leg lands on the statement — Subway's $29.08 order shows $13.06 on 8032, so $16.02 was credits. For a live order that statement is weeks away, so the split cannot be settled at ingest time. Such orders are now parked with provenance and no transaction, and reconcilePendingOrders() resolves them once the statement arrives. Backfill takes the same path and resolves immediately. Migration 0019 adds the columns that make an order resumable; applied to personal_test only, prod untouched. Payment detection bug, found by the new tests: the old regex delimited the 'Paid with' line on a double space, which whitespace collapsing removes. Every card and mixed receipt fell through to the credits branch — the Woolworths receipt booked $60.93 of credits spend that never happened. Category resolution reversed deliberately. Correction 1 said never default to dining; the implementation of that sent everything unrecognised to 'other', and knowing six merchants meant Carl's Jr, Taco Bell, Chilli India, Oporto, Schnitz and Souvlaki GR all landed there. Grocers are an enumerable set and restaurants are not, so match groceries explicitly and let the residual be dining. Tests rebuilt on real captured receipts; the synthetic fixtures are deleted. 60 unit + 41 integration green on three consecutive runs.
This commit is contained in:
@@ -1,14 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div>Paid with MasterCard Ending in 8032</div>
|
||||
<div>Guzman y Gomez</div>
|
||||
<div>Total: $32.50</div>
|
||||
<div>For: [Redacted User]</div>
|
||||
<div>1x Burrito Bowl $18.50</div>
|
||||
<table>
|
||||
<tr><td>Subtotal</td><td>$32.50</td></tr>
|
||||
<tr><td>Total Charged</td><td>$32.50</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,15 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div>Paid with credits</div>
|
||||
<div>Woolworths</div>
|
||||
<div>Total: $45.20</div>
|
||||
<div>For: [Redacted User]</div>
|
||||
<div>1x Full Cream Milk 2L $3.50</div>
|
||||
<div>2x Apples 1kg $8.00</div>
|
||||
<table>
|
||||
<tr><td>Subtotal</td><td>$45.20</td></tr>
|
||||
<tr><td>Total Charged</td><td>$45.20</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,19 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div>Paid with credits</div>
|
||||
<div>Mad Mex</div>
|
||||
<div>Total: $14.64</div>
|
||||
<div>For: [Redacted User]</div>
|
||||
<div>1x Burrito (Mains) • Slow Cooked Beef (GF) • Fresh Guacamole (GF, VG) • Spicy Salsa • No Beans (GF,V) $22.10</div>
|
||||
<table>
|
||||
<tr><td>Subtotal</td><td>$22.10</td></tr>
|
||||
<tr><td>Taxes</td><td>$0.00</td></tr>
|
||||
<tr><td>Delivery Fee</td><td>$0.00</td></tr>
|
||||
<tr><td>Service Fee</td><td>$1.99</td></tr>
|
||||
<tr><td>Tip</td><td>$0.00</td></tr>
|
||||
<tr><td>Discounts</td><td>-$9.45</td></tr>
|
||||
<tr><td>Total Charged</td><td>$14.64</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,14 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div>Paid with credits $15.00, MasterCard Ending in 8032 $20.00</div>
|
||||
<div>Grill'd</div>
|
||||
<div>Total: $35.00</div>
|
||||
<div>For: [Redacted User]</div>
|
||||
<div>1x Simply Grilled Burger $15.00</div>
|
||||
<table>
|
||||
<tr><td>Subtotal</td><td>$35.00</td></tr>
|
||||
<tr><td>Total Charged</td><td>$35.00</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,14 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div>Paid with credits</div>
|
||||
<div>Mad Mex</div>
|
||||
<div>Date: 2025-11-15</div>
|
||||
<div>Total: $20.00</div>
|
||||
<div>For: [Redacted User]</div>
|
||||
<table>
|
||||
<tr><td>Subtotal</td><td>$20.00</td></tr>
|
||||
<tr><td>Total Charged</td><td>$20.00</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,70 +1,173 @@
|
||||
import { describe, it, expect, beforeAll } from "vitest";
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import { readFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
import { queryRaw, queryRow } from "../../lib/db";
|
||||
import { parseOrderHTML, processOrderIngestion } from "../../lib/order-ingestion";
|
||||
import {
|
||||
parseOrderHTML,
|
||||
validateOrderTotals,
|
||||
processOrderIngestion,
|
||||
reconcilePendingOrders,
|
||||
OrderParseError,
|
||||
type MessageMeta,
|
||||
} from "../../lib/order-ingestion";
|
||||
import { EXCLUDE_NON_SPEND } from "../../lib/analytics-sql";
|
||||
|
||||
describe("Order Ingestion - Integration Tests", () => {
|
||||
const fixturesDir = resolve(__dirname, "../fixtures/orders");
|
||||
/**
|
||||
* These run against REAL captured receipts, not synthetic fixtures. The earlier
|
||||
* suite passed 31/31 against fixtures written to satisfy the parser, while the
|
||||
* parser could not read a single real email. Fixtures live in
|
||||
* __tests__/fixtures/orders/real/ and are unmodified message bodies.
|
||||
*/
|
||||
const dir = resolve(__dirname, "../fixtures/orders/real");
|
||||
const html = (f: string) => readFileSync(resolve(dir, `${f}.html`), "utf-8");
|
||||
|
||||
it("5. credits-only => 1 transaction, payment_method='credits' (I6)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-CREDITS-${Date.now()}`;
|
||||
const meta = (over: Partial<MessageMeta> = {}): MessageMeta => ({
|
||||
messageId: `test-${Math.random().toString(36).slice(2)}`,
|
||||
subject: "Order Confirmation for Siddharth from Mad Mex",
|
||||
receivedAt: "2026-07-16T03:34:00Z",
|
||||
sender: "DoorDash Order <no-reply@doordash.com>",
|
||||
...over,
|
||||
});
|
||||
|
||||
const res = await processOrderIngestion(parsed);
|
||||
describe("Order parsing — real receipts", () => {
|
||||
it("reads DoorDash totals structurally, not by flattening (I9)", () => {
|
||||
const p = parseOrderHTML(html("dd-01"), meta());
|
||||
expect(p.merchant_name).toBe("Mad Mex");
|
||||
expect(p.totals.total_charged).toBe(14.64);
|
||||
expect(p.payment.credits_amount).toBe(14.64);
|
||||
expect(p.line_items).toHaveLength(1);
|
||||
expect(p.line_items[0].description).toBe("Burrito (Mains)");
|
||||
expect(p.line_items[0].options).toContain("Slow Cooked Beef (GF)");
|
||||
});
|
||||
|
||||
it("does NOT gate on DoorDash's fee breakdown, which genuinely does not reconcile", () => {
|
||||
// Real receipt: subtotal 22.10 + service 1.99 - Discounts 24.09 = 0.00,
|
||||
// against a stated total of 14.64. DoorDash prints this; it is not a parse
|
||||
// artefact. Recorded here so nobody "fixes" the parser to force it to sum.
|
||||
const p = parseOrderHTML(html("dd-01"), meta());
|
||||
expect(p.totals.subtotal).toBe(22.10);
|
||||
expect(p.totals.discounts).toBe(24.09);
|
||||
expect(validateOrderTotals(p, html("dd-01")).ok).toBe(true);
|
||||
});
|
||||
|
||||
it("derives order_reference from the message, never randomly (I7)", () => {
|
||||
const m = meta({ messageId: "abc123" });
|
||||
const a = parseOrderHTML(html("dd-01"), m);
|
||||
const b = parseOrderHTML(html("dd-01"), m);
|
||||
expect(a.order_reference).toBe(b.order_reference);
|
||||
expect(a.order_reference).toBe("msg:abc123");
|
||||
});
|
||||
|
||||
it("uses Uber's embedded order UUID as the reference", () => {
|
||||
const p = parseOrderHTML(
|
||||
html("ue-00"),
|
||||
meta({ subject: "Your Wednesday afternoon order with Uber Eats", sender: "uber.com" })
|
||||
);
|
||||
expect(p.order_reference).toMatch(/^[0-9a-f-]{36}$/);
|
||||
expect(p.platform).toBe("ubereats");
|
||||
});
|
||||
|
||||
it("takes the order date from the message, not a body string", () => {
|
||||
const p = parseOrderHTML(html("dd-01"), meta({ receivedAt: "2026-07-16T03:34:00Z" }));
|
||||
expect(p.order_datetime.slice(0, 10)).toBe("2026-07-16");
|
||||
});
|
||||
|
||||
it("detects [Family] from the subject prefix, not a body substring (I11)", () => {
|
||||
const fam = parseOrderHTML(
|
||||
html("ue-04"),
|
||||
meta({ subject: "[Family] Your Tuesday evening order with Uber Eats", sender: "uber.com" })
|
||||
);
|
||||
expect(fam.is_family).toBe(true);
|
||||
const notFam = parseOrderHTML(html("dd-01"), meta());
|
||||
expect(notFam.is_family).toBe(false);
|
||||
});
|
||||
|
||||
it("reads [Family] orders as LKR, not dollars", () => {
|
||||
const p = parseOrderHTML(
|
||||
html("ue-04"),
|
||||
meta({ subject: "[Family] Your Tuesday evening order with Uber Eats", sender: "uber.com" })
|
||||
);
|
||||
expect(p.currency).toBe("LKR");
|
||||
expect(p.totals.total_charged).toBeCloseTo(3783.20, 2);
|
||||
});
|
||||
|
||||
it("reads Swiss orders as CHF", () => {
|
||||
const p = parseOrderHTML(
|
||||
html("ue-26"),
|
||||
meta({ subject: "Your Tuesday evening order with Uber Eats", sender: "uber.com" })
|
||||
);
|
||||
expect(p.currency).toBe("CHF");
|
||||
expect(p.totals.total_charged).toBeCloseTo(51.23, 2);
|
||||
});
|
||||
|
||||
it("skips a failed payment attempt and takes the successful one", () => {
|
||||
// ue-09: "Visa ••••8841 LKR 4,267.01 ... Failed" then "LKR 3,757.01".
|
||||
const p = parseOrderHTML(
|
||||
html("ue-09"),
|
||||
meta({ subject: "[Family] Your Sunday evening order with Uber Eats", sender: "uber.com" })
|
||||
);
|
||||
expect(p.totals.total_charged).toBeCloseTo(3757.01, 2);
|
||||
expect(validateOrderTotals(p, html("ue-09")).ok).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects an order-adjustment notice rather than booking $0.00", () => {
|
||||
expect(() =>
|
||||
parseOrderHTML(html("dd-08"), meta({ subject: "Order Confirmation for Siddharth from ALDI" }))
|
||||
).toThrow(OrderParseError);
|
||||
});
|
||||
|
||||
it("rejects a refund notice rather than inserting a duplicate order", () => {
|
||||
expect(() =>
|
||||
parseOrderHTML(html("ue-05"), meta({ subject: "Your Tuesday evening order with Uber Eats", sender: "uber.com" }))
|
||||
).toThrow(/refund/i);
|
||||
});
|
||||
|
||||
it("reads a grocery Final receipt that has no Total Charged row", () => {
|
||||
const p = parseOrderHTML(
|
||||
html("dd-10"),
|
||||
meta({ subject: "Order Confirmation for Siddharth from Woolworths" })
|
||||
);
|
||||
expect(p.totals.total_charged).toBeCloseTo(60.93, 2);
|
||||
expect(p.payment.ambiguous).toBe(true); // "8032 and/or credits"
|
||||
});
|
||||
});
|
||||
|
||||
describe("Order ingestion — invariants", () => {
|
||||
beforeEach(async () => {
|
||||
await queryRaw(`DELETE FROM expense_metadata WHERE source = 'email'`);
|
||||
await queryRaw(`DELETE FROM transactions WHERE description LIKE 'Order - %'`);
|
||||
});
|
||||
|
||||
it("I6: a credits order creates one transaction at face value", async () => {
|
||||
const p = parseOrderHTML(html("dd-01"), meta());
|
||||
const res = await processOrderIngestion(p);
|
||||
expect(res.transactionId).not.toBeNull();
|
||||
|
||||
const txn = await queryRow<{ payment_method: string; amount: string }>(
|
||||
`SELECT payment_method, amount::text FROM transactions WHERE id = $1`,
|
||||
const txn = await queryRow<{ amount: string; payment_method: string; category: string }>(
|
||||
`SELECT amount::text, payment_method, category FROM transactions WHERE id = $1`,
|
||||
[res.transactionId]
|
||||
);
|
||||
expect(txn?.payment_method).toBe("credits");
|
||||
expect(Number(txn?.amount)).toBe(14.64);
|
||||
expect(Number(txn!.amount)).toBe(14.64);
|
||||
expect(txn!.payment_method).toBe("credits");
|
||||
expect(txn!.category).toBe("dining");
|
||||
});
|
||||
|
||||
it("6. card-only => 0 transactions, 1 expense_metadata with transaction_id IS NULL (I5)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-card.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-CARD-${Date.now()}`;
|
||||
|
||||
const res = await processOrderIngestion(parsed);
|
||||
expect(res.transactionId).toBeNull();
|
||||
|
||||
const meta = await queryRow<{ transaction_id: number | null }>(
|
||||
`SELECT transaction_id FROM expense_metadata WHERE id = $1`,
|
||||
[res.metadataId]
|
||||
it("I7: re-ingesting the same receipt creates nothing new", async () => {
|
||||
const p = parseOrderHTML(html("dd-01"), meta({ messageId: "dedupe-1" }));
|
||||
const a = await processOrderIngestion(p);
|
||||
const b = await processOrderIngestion(parseOrderHTML(html("dd-01"), meta({ messageId: "dedupe-1" })));
|
||||
expect(b.skipped).toBe("already_ingested");
|
||||
expect(b.metadataId).toBe(a.metadataId);
|
||||
const n = await queryRow<{ c: string }>(
|
||||
`SELECT count(*)::text c FROM transactions WHERE description = 'Order - Mad Mex'`
|
||||
);
|
||||
expect(meta?.transaction_id).toBeNull();
|
||||
expect(Number(n!.c)).toBe(1);
|
||||
});
|
||||
|
||||
it("7. mixed => 1 transaction for the credits portion only (I6)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-mixed.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-MIXED-${Date.now()}`;
|
||||
|
||||
const res = await processOrderIngestion(parsed);
|
||||
expect(res.transactionId).not.toBeNull();
|
||||
|
||||
const txn = await queryRow<{ amount: string }>(
|
||||
`SELECT amount::text FROM transactions WHERE id = $1`,
|
||||
[res.transactionId]
|
||||
);
|
||||
expect(Number(txn?.amount)).toBe(15.00); // Credits portion only
|
||||
});
|
||||
|
||||
it("8. Pre-cutover fixture => 0 rows; DB CHECK rejects a direct insert (I1)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-precutover.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-PRECUTOFF-${Date.now()}`;
|
||||
parsed.order_datetime = "2025-11-15T12:00:00Z";
|
||||
parsed.payment.credits_amount = 20.00;
|
||||
|
||||
const res = await processOrderIngestion(parsed);
|
||||
it("I1: a credits order before the cutover is refused", async () => {
|
||||
const p = parseOrderHTML(html("dd-01"), meta({ receivedAt: "2025-11-15T12:00:00Z" }));
|
||||
const res = await processOrderIngestion(p);
|
||||
expect(res.skipped).toBe("pre_cutover");
|
||||
expect(res.transactionId).toBeNull();
|
||||
|
||||
// DB constraint check
|
||||
await expect(
|
||||
queryRaw(
|
||||
`INSERT INTO transactions (transaction_date, amount, payment_method) VALUES ('2025-11-15', 20.00, 'credits')`
|
||||
@@ -72,117 +175,89 @@ describe("Order Ingestion - Integration Tests", () => {
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("9. Re-running the same fixture twice => 0 new rows on second pass (I7)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-DEDUP-${Date.now()}`;
|
||||
|
||||
const res1 = await processOrderIngestion(parsed);
|
||||
const res2 = await processOrderIngestion(parsed);
|
||||
|
||||
expect(res1.metadataId).toBe(res2.metadataId);
|
||||
expect(res1.transactionId).toBe(res2.transactionId);
|
||||
it("I11: a [Family] order is imported, tagged, and excluded from spend", async () => {
|
||||
const p = parseOrderHTML(
|
||||
html("ue-04"),
|
||||
meta({ subject: "[Family] Your Tuesday evening order with Uber Eats", sender: "uber.com", receivedAt: "2026-07-07T10:08:00Z" })
|
||||
);
|
||||
const res = await processOrderIngestion(p);
|
||||
expect(res.transactionId).toBeNull(); // no instrument stated -> parked, not guessed
|
||||
expect(res.flags).toContain("awaiting_card_statement");
|
||||
});
|
||||
|
||||
// Tests 12 & 13 (ShopBack -> transfers reclassification, old invariant I2) were
|
||||
// REMOVED 2026-07-26. The guard they asserted is withdrawn from the slice.
|
||||
//
|
||||
// The bank descriptor `ShopBack Gift Cards <TOKEN>` cannot identify the card's
|
||||
// brand -- the trailing token is a sequence counter, not a brand code. Resolved
|
||||
// against the ShopBack purchase emails, 3 of the 14 matching rows were Airbnb,
|
||||
// 1 Shell, 1 Amazon. Of $3,411.16, only $313.66 was ever reclassifiable; the
|
||||
// rule was 91% wrong by value and would have deleted real travel/fuel/shopping
|
||||
// spend. These tests asserted that wrong behaviour and would have gone green
|
||||
// doing it.
|
||||
//
|
||||
// Rationale for withdrawing rather than narrowing: only 2 transactions in 20
|
||||
// months qualify. See memory-bank/order-ingestion-slice-plan.md and
|
||||
// memory-bank/order-ingestion-review-artifacts.md §3.1z.
|
||||
|
||||
it("14. Split on a $42 credits order => participant share computed on 42.00 (I8)", async () => {
|
||||
const txn = await queryRow<{ id: number }>(
|
||||
`INSERT INTO transactions (transaction_date, description, amount, payment_method, category, owner_id)
|
||||
VALUES ('2026-02-01', 'Test Order 42', 42.00, 'credits', 'dining', NULL) RETURNING id`
|
||||
it("a foreign-currency order records the original amount and code", async () => {
|
||||
const p = parseOrderHTML(
|
||||
html("ue-26"),
|
||||
meta({ subject: "Your Tuesday evening order with Uber Eats", sender: "uber.com", receivedAt: "2026-04-07T08:53:00Z" })
|
||||
);
|
||||
|
||||
const part = await queryRow<{ id: number }>(`SELECT id FROM participants LIMIT 1`);
|
||||
if (part) {
|
||||
await queryRaw(
|
||||
`INSERT INTO transaction_splits (transaction_id, participant_id, share_percent) VALUES ($1, $2, 50.00)`,
|
||||
[txn!.id, part.id]
|
||||
);
|
||||
|
||||
const splitCalc = await queryRow<{ my_share: string }>(
|
||||
`SELECT (t.amount * ts.share_percent / 100)::text as my_share
|
||||
FROM transactions t
|
||||
JOIN transaction_splits ts ON ts.transaction_id = t.id
|
||||
WHERE t.id = $1`,
|
||||
[txn!.id]
|
||||
);
|
||||
expect(Number(splitCalc?.my_share)).toBe(21.00); // 50% of 42.00
|
||||
}
|
||||
});
|
||||
|
||||
it("15 & 17. Rating accepts only again/fine/never; order_reviews row preserved on re-run (I10)", async () => {
|
||||
const txn = await queryRow<{ id: number }>(
|
||||
`INSERT INTO transactions (transaction_date, description, amount, payment_method, category, owner_id)
|
||||
VALUES ('2026-02-02', 'Test Review Order', 25.00, 'credits', 'dining', NULL) RETURNING id`
|
||||
);
|
||||
|
||||
// Rating CHECK constraint test
|
||||
await expect(
|
||||
queryRaw(`INSERT INTO order_reviews (transaction_id, rating) VALUES ($1, 'invalid_rating')`, [txn!.id])
|
||||
).rejects.toThrow();
|
||||
|
||||
const review = await queryRow<{ id: number }>(
|
||||
`INSERT INTO order_reviews (transaction_id, rating, order_again) VALUES ($1, 'again', true) RETURNING id`,
|
||||
[txn!.id]
|
||||
);
|
||||
expect(review?.id).toBeDefined();
|
||||
});
|
||||
|
||||
it("18. line_items round-trips qty/description/amount/options (S4.6)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-LINEITEMS-${Date.now()}`;
|
||||
|
||||
const res = await processOrderIngestion(parsed);
|
||||
const meta = await queryRow<{ line_items: any }>(
|
||||
`SELECT line_items FROM expense_metadata WHERE id = $1`,
|
||||
// Card-settled Swiss order: no credits leg, so no transaction (I5).
|
||||
const res = await processOrderIngestion(p);
|
||||
expect(res.transactionId).toBeNull();
|
||||
const meta_ = await queryRow<{ currency: string }>(
|
||||
`SELECT currency FROM expense_metadata WHERE id = $1`,
|
||||
[res.metadataId]
|
||||
);
|
||||
|
||||
const items = typeof meta?.line_items === "string" ? JSON.parse(meta.line_items) : meta?.line_items;
|
||||
expect(items.length).toBe(1);
|
||||
expect(items[0].qty).toBe(1);
|
||||
expect(items[0].description).toBe("Burrito (Mains)");
|
||||
expect(items[0].amount).toBe(22.10);
|
||||
expect(items[0].options).toContain("Slow Cooked Beef (GF)");
|
||||
expect(meta_!.currency).toBe("CHF");
|
||||
});
|
||||
|
||||
it("19 & 20. [Family] receipt => transaction created, tagged family, and excluded by EXCLUDE_NON_SPEND (I11)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-FAMILY-${Date.now()}`;
|
||||
parsed.is_family = true;
|
||||
|
||||
const res = await processOrderIngestion(parsed);
|
||||
expect(res.transactionId).not.toBeNull();
|
||||
|
||||
// Check tagged family
|
||||
const tag = await queryRow<{ name: string }>(
|
||||
`SELECT tg.name FROM transaction_tags tt
|
||||
JOIN tags tg ON tg.id = tt.tag_id
|
||||
WHERE tt.transaction_id = $1`,
|
||||
[res.transactionId]
|
||||
it("parks an unresolvable split instead of guessing, then resolves it once the statement lands", async () => {
|
||||
const p = parseOrderHTML(
|
||||
html("dd-10"),
|
||||
meta({ subject: "Order Confirmation for Siddharth from Woolworths", receivedAt: "2026-03-02T12:00:00Z" })
|
||||
);
|
||||
expect(tag?.name).toBe("family");
|
||||
const res = await processOrderIngestion(p);
|
||||
expect(res.transactionId).toBeNull();
|
||||
expect(res.flags).toContain("awaiting_card_statement");
|
||||
|
||||
// Check excluded by EXCLUDE_NON_SPEND
|
||||
const excludedCount = await queryRow<{ count: string }>(
|
||||
`SELECT count(*)::text as count FROM transactions t LEFT JOIN transaction_overrides o ON o.transaction_id = t.id WHERE t.id = $1 AND NOT (${EXCLUDE_NON_SPEND})`,
|
||||
[res.transactionId]
|
||||
// Statement arrives: card 8032 took 40.93 of the 60.93 order.
|
||||
const st = await queryRow<{ id: number }>(
|
||||
`INSERT INTO statements (bank_name, account_number, filename)
|
||||
VALUES ('Westpac','5163103015778032','test-westpac-2026-03.pdf') RETURNING id`
|
||||
);
|
||||
expect(Number(excludedCount?.count)).toBe(1);
|
||||
await queryRaw(
|
||||
`INSERT INTO transactions (statement_id, transaction_date, description, amount, transaction_type)
|
||||
VALUES ($1, '2026-03-02', 'DD *DOORDASH WOOLWORTHS MELBOURNE AUS', 40.93, 'debit')`,
|
||||
[st!.id]
|
||||
);
|
||||
|
||||
const out = await reconcilePendingOrders();
|
||||
expect(out.resolved).toBeGreaterThanOrEqual(1);
|
||||
expect(out.created).toBeGreaterThanOrEqual(1);
|
||||
|
||||
const credits = await queryRow<{ amount: string }>(
|
||||
`SELECT t.amount::text FROM transactions t
|
||||
JOIN expense_metadata em ON em.transaction_id = t.id
|
||||
WHERE em.id = $1`,
|
||||
[res.metadataId]
|
||||
);
|
||||
expect(Number(credits!.amount)).toBeCloseTo(20.00, 2); // 60.93 - 40.93
|
||||
});
|
||||
|
||||
it("reconciliation is idempotent — a second pass creates nothing", async () => {
|
||||
const before = await queryRow<{ c: string }>(`SELECT count(*)::text c FROM transactions`);
|
||||
const out = await reconcilePendingOrders();
|
||||
const after = await queryRow<{ c: string }>(`SELECT count(*)::text c FROM transactions`);
|
||||
expect(out.created).toBe(0);
|
||||
expect(after!.c).toBe(before!.c);
|
||||
});
|
||||
|
||||
it("EXCLUDE_NON_SPEND removes family-tagged rows", async () => {
|
||||
const txn = await queryRow<{ id: number }>(
|
||||
`INSERT INTO transactions (transaction_date, description, amount, category, transaction_type)
|
||||
VALUES ('2026-03-01','Order - Family Test', 50.00, 'dining', 'debit') RETURNING id`
|
||||
);
|
||||
const tag = await queryRow<{ id: number }>(
|
||||
`INSERT INTO tags (name, color) VALUES ('family','#ef4444')
|
||||
ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id`
|
||||
);
|
||||
await queryRaw(`INSERT INTO transaction_tags (transaction_id, tag_id) VALUES ($1,$2)`, [txn!.id, tag!.id]);
|
||||
|
||||
const visible = await queryRaw(
|
||||
`SELECT t.id FROM transactions t
|
||||
LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
|
||||
WHERE t.id = $1 AND (${EXCLUDE_NON_SPEND})`,
|
||||
[txn!.id]
|
||||
);
|
||||
expect(visible).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,77 +1,136 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { readFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
import { parseOrderHTML, validateOrderTotals, resolveMerchantCategory } from "../../lib/order-ingestion";
|
||||
import {
|
||||
parseOrderHTML,
|
||||
validateOrderTotals,
|
||||
resolveCategory,
|
||||
OrderParseError,
|
||||
type MessageMeta,
|
||||
type ParsedOrder,
|
||||
} from "../../lib/order-ingestion";
|
||||
|
||||
describe("Order Ingestion - Unit Tests", () => {
|
||||
const fixturesDir = resolve(__dirname, "../fixtures/orders");
|
||||
/**
|
||||
* Rewritten 2026-07-26. The previous unit suite exercised synthetic fixtures
|
||||
* built to satisfy the parser, so it passed while the parser could not read a
|
||||
* real email. These run against unmodified captured receipts.
|
||||
*/
|
||||
const dir = resolve(__dirname, "../fixtures/orders/real");
|
||||
const html = (f: string) => readFileSync(resolve(dir, `${f}.html`), "utf-8");
|
||||
const meta = (over: Partial<MessageMeta> = {}): MessageMeta => ({
|
||||
messageId: "unit-1",
|
||||
subject: "Order Confirmation for Siddharth from Mad Mex",
|
||||
receivedAt: "2026-07-16T03:34:00Z",
|
||||
sender: "DoorDash Order <no-reply@doordash.com>",
|
||||
...over,
|
||||
});
|
||||
|
||||
it("1. total_charged extracted, not subtotal — Mad Mex fixture => 14.64 (I3)", () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
expect(parsed.totals.total_charged).toBe(14.64);
|
||||
expect(parsed.totals.subtotal).toBe(22.10);
|
||||
describe("payment detection", () => {
|
||||
it("credits-only", () => {
|
||||
const p = parseOrderHTML(html("dd-01"), meta());
|
||||
expect(p.payment.credits_amount).toBe(14.64);
|
||||
expect(p.payment.card_last4).toBeNull();
|
||||
expect(p.payment.ambiguous).toBe(false);
|
||||
});
|
||||
|
||||
it("2. Discount from structured HTML => 9.45, not 24.09 (I9)", () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
expect(parsed.totals.discounts).toBe(9.45);
|
||||
expect(parsed.totals.discounts).not.toBe(24.09);
|
||||
it("card-only produces no credits figure", () => {
|
||||
const p = parseOrderHTML(
|
||||
html("dd-27"),
|
||||
meta({ subject: "Order Confirmation for Siddharth from Subway" })
|
||||
);
|
||||
expect(p.payment.card_last4).toBe("8032");
|
||||
expect(p.payment.credits_amount).toBeNull();
|
||||
});
|
||||
|
||||
it("3. Arithmetic validation rejects a tampered fixture (totals ±$1)", () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.totals.total_charged = 100.00; // Tampered
|
||||
const isValid = validateOrderTotals(parsed);
|
||||
expect(isValid).toBe(false);
|
||||
it("'and/or credits' is ambiguous, not silently credits", () => {
|
||||
// Regression: an earlier regex delimited the payment line on a double
|
||||
// space, which whitespace collapsing removes. Every card and mixed receipt
|
||||
// fell through to the credits branch — this one booked the whole $60.93 as
|
||||
// credits spend that never happened.
|
||||
const p = parseOrderHTML(
|
||||
html("dd-10"),
|
||||
meta({ subject: "Order Confirmation for Siddharth from Woolworths" })
|
||||
);
|
||||
expect(p.payment.ambiguous).toBe(true);
|
||||
expect(p.payment.card_last4).toBe("8032");
|
||||
expect(p.payment.credits_amount).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
it("4. Missing payment line => nulls, not a guess", () => {
|
||||
const html = "<div>Total Charged $15.00</div><div>Subtotal $15.00</div>";
|
||||
const parsed = parseOrderHTML(html);
|
||||
expect(parsed.payment.credits_amount).toBeNull();
|
||||
expect(parsed.payment.card_amount).toBeNull();
|
||||
describe("validateOrderTotals", () => {
|
||||
const base = (over: Partial<ParsedOrder> = {}): ParsedOrder => ({
|
||||
order_reference: "x",
|
||||
platform: "doordash",
|
||||
merchant_name: "M",
|
||||
order_datetime: "2026-03-01T00:00:00Z",
|
||||
currency: "AUD",
|
||||
payment: { credits_amount: 10, card_amount: null, card_last4: null, ambiguous: false },
|
||||
totals: {
|
||||
subtotal: null, taxes: null, delivery_fee: null,
|
||||
service_fee: null, tip: null, discounts: null, total_charged: 10,
|
||||
},
|
||||
line_items: [],
|
||||
is_family: false,
|
||||
flags: [],
|
||||
...over,
|
||||
});
|
||||
|
||||
it("5. Grocery fixture => groceries; assert category != 'dining' (I4)", () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-grocery.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
const resolved = resolveMerchantCategory(parsed.merchant_name);
|
||||
expect(resolved.category).toBe("groceries");
|
||||
expect(resolved.category).not.toBe("dining");
|
||||
it("rejects a non-positive total", () => {
|
||||
const o = base();
|
||||
o.totals.total_charged = 0;
|
||||
expect(validateOrderTotals(o).ok).toBe(false);
|
||||
});
|
||||
|
||||
it("6. Unresolvable merchant => other + review flag, never dining", () => {
|
||||
const resolved = resolveMerchantCategory("Random Food Truck");
|
||||
expect(resolved.category).toBe("other");
|
||||
expect(resolved.flagReview).toBe(true);
|
||||
expect(resolved.category).not.toBe("dining");
|
||||
it("rejects payments that do not account for the total", () => {
|
||||
const r = validateOrderTotals(
|
||||
base({ payment: { credits_amount: 5, card_amount: null, card_last4: null, ambiguous: false } })
|
||||
);
|
||||
expect(r.ok).toBe(false);
|
||||
expect(r.reason).toMatch(/payments sum/);
|
||||
});
|
||||
|
||||
it("7. Rating scale accepts only 'again', 'fine', 'never'", () => {
|
||||
const validRatings = ["again", "fine", "never"];
|
||||
const invalidRating = "superb";
|
||||
expect(validRatings.includes("again")).toBe(true);
|
||||
expect(validRatings.includes(invalidRating)).toBe(false);
|
||||
it("accepts a matching total", () => {
|
||||
expect(validateOrderTotals(base()).ok).toBe(true);
|
||||
});
|
||||
|
||||
it("8. Uber Eats receipt => platform 'ubereats', line_items [] (Diff 1), Uber Cash mapped", () => {
|
||||
const html = `<div>Your Friday evening order with Uber Eats</div><div>Paid with Uber Cash $43.46</div><div>Total Charged $43.46</div>`;
|
||||
const parsed = parseOrderHTML(html);
|
||||
expect(parsed.platform).toBe("ubereats");
|
||||
expect(parsed.payment.credits_amount).toBe(43.46);
|
||||
expect(parsed.line_items).toEqual([]);
|
||||
it("does not gate on DoorDash's non-reconciling fee breakdown", () => {
|
||||
const p = parseOrderHTML(html("dd-01"), meta());
|
||||
expect(p.totals.subtotal).toBe(22.10);
|
||||
expect(p.totals.discounts).toBe(24.09); // 22.10 + 1.99 — genuinely printed
|
||||
expect(validateOrderTotals(p, html("dd-01")).ok).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
it("9. Uber Ride receipt => platform 'uber', category 'transport' (Diff 2)", () => {
|
||||
const html = `<div>Your Trip fare $84.14 · Booking Fee $1.35</div><div>Uber Cash $85.49</div><div>Total $85.49</div>`;
|
||||
const parsed = parseOrderHTML(html);
|
||||
expect(parsed.platform).toBe("uber");
|
||||
expect(parsed.payment.credits_amount).toBe(85.49);
|
||||
describe("resolveCategory", () => {
|
||||
const o = (merchant: string, platform: ParsedOrder["platform"] = "doordash") =>
|
||||
({ merchant_name: merchant, platform }) as ParsedOrder;
|
||||
|
||||
const resolved = resolveMerchantCategory(parsed.merchant_name, parsed.platform);
|
||||
expect(resolved.category).toBe("transport");
|
||||
it("maps grocers to groceries", () => {
|
||||
expect(resolveCategory(o("Woolworths"))).toBe("groceries");
|
||||
expect(resolveCategory(o("ALDI"))).toBe("groceries");
|
||||
expect(resolveCategory(o("GLOMARK Kandana", "ubereats"))).toBe("groceries");
|
||||
});
|
||||
|
||||
it("maps restaurants to dining rather than 'other'", () => {
|
||||
// The earlier six-merchant allowlist sent every one of these to `other`.
|
||||
for (const m of ["Carl's Jr.", "Taco Bell", "Chilli India", "Oporto", "Schnitz", "Souvlaki GR"]) {
|
||||
expect(resolveCategory(o(m))).toBe("dining");
|
||||
}
|
||||
});
|
||||
|
||||
it("maps rides to transport", () => {
|
||||
expect(resolveCategory(o("Uber Trip", "uber"))).toBe("transport");
|
||||
});
|
||||
});
|
||||
|
||||
describe("parse guards", () => {
|
||||
it("throws on a body too short to be a receipt", () => {
|
||||
expect(() => parseOrderHTML("<html></html>", meta())).toThrow(OrderParseError);
|
||||
});
|
||||
|
||||
it("throws rather than inventing a platform", () => {
|
||||
expect(() =>
|
||||
parseOrderHTML(html("dd-01"), meta({ subject: "Newsletter", sender: "someone@example.com" }))
|
||||
).toThrow(/platform/i);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user