feat(orders): implement order ingestion pipeline, review ratings schema, and [Family] exclusion

2026-07-26 18:52:34 +10:00
parent d06088fe34
commit bbb90238e4
13 changed files with 667 additions and 3 deletions
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<body>
+  <div>Paid with MasterCard Ending in 8032</div>
+  <div>Guzman y Gomez</div>
+  <div>Total: $32.50</div>
+  <div>For: [Redacted User]</div>
+  <div>1x Burrito Bowl $18.50</div>
+  <table>
+    <tr><td>Subtotal</td><td>$32.50</td></tr>
+    <tr><td>Total Charged</td><td>$32.50</td></tr>
+  </table>
+</body>
+</html>
@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+<body>
+  <div>Paid with credits</div>
+  <div>Woolworths</div>
+  <div>Total: $45.20</div>
+  <div>For: [Redacted User]</div>
+  <div>1x Full Cream Milk 2L $3.50</div>
+  <div>2x Apples 1kg $8.00</div>
+  <table>
+    <tr><td>Subtotal</td><td>$45.20</td></tr>
+    <tr><td>Total Charged</td><td>$45.20</td></tr>
+  </table>
+</body>
+</html>
@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html>
+<body>
+  <div>Paid with credits</div>
+  <div>Mad Mex</div>
+  <div>Total: $14.64</div>
+  <div>For: [Redacted User]</div>
+  <div>1x Burrito (Mains) • Slow Cooked Beef (GF) • Fresh Guacamole (GF, VG) • Spicy Salsa • No Beans (GF,V) $22.10</div>
+  <table>
+    <tr><td>Subtotal</td><td>$22.10</td></tr>
+    <tr><td>Taxes</td><td>$0.00</td></tr>
+    <tr><td>Delivery Fee</td><td>$0.00</td></tr>
+    <tr><td>Service Fee</td><td>$1.99</td></tr>
+    <tr><td>Tip</td><td>$0.00</td></tr>
+    <tr><td>Discounts</td><td>-$9.45</td></tr>
+    <tr><td>Total Charged</td><td>$14.64</td></tr>
+  </table>
+</body>
+</html>
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<body>
+  <div>Paid with credits $15.00, MasterCard Ending in 8032 $20.00</div>
+  <div>Grill'd</div>
+  <div>Total: $35.00</div>
+  <div>For: [Redacted User]</div>
+  <div>1x Simply Grilled Burger $15.00</div>
+  <table>
+    <tr><td>Subtotal</td><td>$35.00</td></tr>
+    <tr><td>Total Charged</td><td>$35.00</td></tr>
+  </table>
+</body>
+</html>
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<body>
+  <div>Paid with credits</div>
+  <div>Mad Mex</div>
+  <div>Date: 2025-11-15</div>
+  <div>Total: $20.00</div>
+  <div>For: [Redacted User]</div>
+  <table>
+    <tr><td>Subtotal</td><td>$20.00</td></tr>
+    <tr><td>Total Charged</td><td>$20.00</td></tr>
+  </table>
+</body>
+</html>
@@ -0,0 +1,206 @@
+import { describe, it, expect, beforeAll } from "vitest";
+import { readFileSync } from "fs";
+import { resolve } from "path";
+import { queryRaw, queryRow } from "../../lib/db";
+import { parseOrderHTML, processOrderIngestion } from "../../lib/order-ingestion";
+import { EXCLUDE_NON_SPEND } from "../../lib/analytics-sql";
+
+describe("Order Ingestion - Integration Tests", () => {
+  const fixturesDir = resolve(__dirname, "../fixtures/orders");
+
+  it("5. credits-only => 1 transaction, payment_method='credits' (I6)", async () => {
+    const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
+    const parsed = parseOrderHTML(html);
+    parsed.order_reference = `TEST-CREDITS-${Date.now()}`;
+
+    const res = await processOrderIngestion(parsed);
+    expect(res.transactionId).not.toBeNull();
+
+    const txn = await queryRow<{ payment_method: string; amount: string }>(
+      `SELECT payment_method, amount::text FROM transactions WHERE id = $1`,
+      [res.transactionId]
+    );
+    expect(txn?.payment_method).toBe("credits");
+    expect(Number(txn?.amount)).toBe(14.64);
+  });
+
+  it("6. card-only => 0 transactions, 1 expense_metadata with transaction_id IS NULL (I5)", async () => {
+    const html = readFileSync(resolve(fixturesDir, "doordash-card.html"), "utf-8");
+    const parsed = parseOrderHTML(html);
+    parsed.order_reference = `TEST-CARD-${Date.now()}`;
+
+    const res = await processOrderIngestion(parsed);
+    expect(res.transactionId).toBeNull();
+
+    const meta = await queryRow<{ transaction_id: number | null }>(
+      `SELECT transaction_id FROM expense_metadata WHERE id = $1`,
+      [res.metadataId]
+    );
+    expect(meta?.transaction_id).toBeNull();
+  });
+
+  it("7. mixed => 1 transaction for the credits portion only (I6)", async () => {
+    const html = readFileSync(resolve(fixturesDir, "doordash-mixed.html"), "utf-8");
+    const parsed = parseOrderHTML(html);
+    parsed.order_reference = `TEST-MIXED-${Date.now()}`;
+
+    const res = await processOrderIngestion(parsed);
+    expect(res.transactionId).not.toBeNull();
+
+    const txn = await queryRow<{ amount: string }>(
+      `SELECT amount::text FROM transactions WHERE id = $1`,
+      [res.transactionId]
+    );
+    expect(Number(txn?.amount)).toBe(15.00); // Credits portion only
+  });
+
+  it("8. Pre-cutover fixture => 0 rows; DB CHECK rejects a direct insert (I1)", async () => {
+    const html = readFileSync(resolve(fixturesDir, "doordash-precutover.html"), "utf-8");
+    const parsed = parseOrderHTML(html);
+    parsed.order_reference = `TEST-PRECUTOFF-${Date.now()}`;
+    parsed.order_datetime = "2025-11-15T12:00:00Z";
+    parsed.payment.credits_amount = 20.00;
+
+    const res = await processOrderIngestion(parsed);
+    expect(res.transactionId).toBeNull();
+
+    // DB constraint check
+    await expect(
+      queryRaw(
+        `INSERT INTO transactions (transaction_date, amount, payment_method) VALUES ('2025-11-15', 20.00, 'credits')`
+      )
+    ).rejects.toThrow();
+  });
+
+  it("9. Re-running the same fixture twice => 0 new rows on second pass (I7)", async () => {
+    const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
+    const parsed = parseOrderHTML(html);
+    parsed.order_reference = `TEST-DEDUP-${Date.now()}`;
+
+    const res1 = await processOrderIngestion(parsed);
+    const res2 = await processOrderIngestion(parsed);
+
+    expect(res1.metadataId).toBe(res2.metadataId);
+    expect(res1.transactionId).toBe(res2.transactionId);
+  });
+
+  it("12 & 13. ShopBack rows carry category_override='transfers' and monthly spend delta matches (I2)", async () => {
+    // Ensure at least one ShopBack transaction exists for isolated test runs
+    await queryRaw(`
+      INSERT INTO transactions (transaction_date, description, amount, category, owner_id)
+      VALUES ('2025-01-05', 'ShopBack Gift Cards SP Australia AUS', 300.00, 'gifts', NULL)
+      ON CONFLICT DO NOTHING;
+    `);
+
+    await queryRaw(`
+      INSERT INTO transaction_overrides (transaction_id, category_override)
+      SELECT id, 'transfers' FROM transactions WHERE description ILIKE '%ShopBack Gift Cards%'
+      ON CONFLICT (transaction_id) DO UPDATE SET category_override = 'transfers';
+    `);
+
+    const sbRows = await queryRaw<{ id: number; category_override: string }>(
+      `SELECT t.id, o.category_override
+       FROM transactions t
+       JOIN transaction_overrides o ON o.transaction_id = t.id
+       WHERE t.description ILIKE '%ShopBack Gift Cards%'`
+    );
+    expect(sbRows.length).toBeGreaterThan(0);
+    sbRows.forEach((r) => expect(r.category_override).toBe("transfers"));
+
+    // Verify EXCLUDE_NON_SPEND excludes them
+    const excluded = await queryRaw(
+      `SELECT t.id FROM transactions t
+       LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
+       WHERE t.description ILIKE '%ShopBack Gift Cards%'
+         AND NOT (${EXCLUDE_NON_SPEND})`
+    );
+    expect(excluded.length).toBe(sbRows.length);
+  });
+
+  it("14. Split on a $42 credits order => participant share computed on 42.00 (I8)", async () => {
+    const txn = await queryRow<{ id: number }>(
+      `INSERT INTO transactions (transaction_date, description, amount, payment_method, category, owner_id)
+       VALUES ('2026-02-01', 'Test Order 42', 42.00, 'credits', 'dining', NULL) RETURNING id`
+    );
+
+    const part = await queryRow<{ id: number }>(`SELECT id FROM participants LIMIT 1`);
+    if (part) {
+      await queryRaw(
+        `INSERT INTO transaction_splits (transaction_id, participant_id, share_percent) VALUES ($1, $2, 50.00)`,
+        [txn!.id, part.id]
+      );
+
+      const splitCalc = await queryRow<{ my_share: string }>(
+        `SELECT (t.amount * ts.share_percent / 100)::text as my_share
+         FROM transactions t
+         JOIN transaction_splits ts ON ts.transaction_id = t.id
+         WHERE t.id = $1`,
+        [txn!.id]
+      );
+      expect(Number(splitCalc?.my_share)).toBe(21.00); // 50% of 42.00
+    }
+  });
+
+  it("15 & 17. Rating accepts only again/fine/never; order_reviews row preserved on re-run (I10)", async () => {
+    const txn = await queryRow<{ id: number }>(
+      `INSERT INTO transactions (transaction_date, description, amount, payment_method, category, owner_id)
+       VALUES ('2026-02-02', 'Test Review Order', 25.00, 'credits', 'dining', NULL) RETURNING id`
+    );
+
+    // Rating CHECK constraint test
+    await expect(
+      queryRaw(`INSERT INTO order_reviews (transaction_id, rating) VALUES ($1, 'invalid_rating')`, [txn!.id])
+    ).rejects.toThrow();
+
+    const review = await queryRow<{ id: number }>(
+      `INSERT INTO order_reviews (transaction_id, rating, order_again) VALUES ($1, 'again', true) RETURNING id`,
+      [txn!.id]
+    );
+    expect(review?.id).toBeDefined();
+  });
+
+  it("18. line_items round-trips qty/description/amount/options (S4.6)", async () => {
+    const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
+    const parsed = parseOrderHTML(html);
+    parsed.order_reference = `TEST-LINEITEMS-${Date.now()}`;
+
+    const res = await processOrderIngestion(parsed);
+    const meta = await queryRow<{ line_items: any }>(
+      `SELECT line_items FROM expense_metadata WHERE id = $1`,
+      [res.metadataId]
+    );
+
+    const items = typeof meta?.line_items === "string" ? JSON.parse(meta.line_items) : meta?.line_items;
+    expect(items.length).toBe(1);
+    expect(items[0].qty).toBe(1);
+    expect(items[0].description).toBe("Burrito (Mains)");
+    expect(items[0].amount).toBe(22.10);
+    expect(items[0].options).toContain("Slow Cooked Beef (GF)");
+  });
+
+  it("19 & 20. [Family] receipt => transaction created, tagged family, and excluded by EXCLUDE_NON_SPEND (I11)", async () => {
+    const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
+    const parsed = parseOrderHTML(html);
+    parsed.order_reference = `TEST-FAMILY-${Date.now()}`;
+    parsed.is_family = true;
+
+    const res = await processOrderIngestion(parsed);
+    expect(res.transactionId).not.toBeNull();
+
+    // Check tagged family
+    const tag = await queryRow<{ name: string }>(
+      `SELECT tg.name FROM transaction_tags tt
+       JOIN tags tg ON tg.id = tt.tag_id
+       WHERE tt.transaction_id = $1`,
+      [res.transactionId]
+    );
+    expect(tag?.name).toBe("family");
+
+    // Check excluded by EXCLUDE_NON_SPEND
+    const excludedCount = await queryRow<{ count: string }>(
+      `SELECT count(*)::text as count FROM transactions t LEFT JOIN transaction_overrides o ON o.transaction_id = t.id WHERE t.id = $1 AND NOT (${EXCLUDE_NON_SPEND})`,
+      [res.transactionId]
+    );
+    expect(Number(excludedCount?.count)).toBe(1);
+  });
+});
@@ -0,0 +1,59 @@
+import { describe, it, expect } from "vitest";
+import { readFileSync } from "fs";
+import { resolve } from "path";
+import { parseOrderHTML, validateOrderTotals, resolveMerchantCategory } from "../../lib/order-ingestion";
+
+describe("Order Ingestion - Unit Tests", () => {
+  const fixturesDir = resolve(__dirname, "../fixtures/orders");
+
+  it("1. total_charged extracted, not subtotal — Mad Mex fixture => 14.64 (I3)", () => {
+    const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
+    const parsed = parseOrderHTML(html);
+    expect(parsed.totals.total_charged).toBe(14.64);
+    expect(parsed.totals.subtotal).toBe(22.10);
+  });
+
+  it("2. Discount from structured HTML => 9.45, not 24.09 (I9)", () => {
+    const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
+    const parsed = parseOrderHTML(html);
+    expect(parsed.totals.discounts).toBe(9.45);
+    expect(parsed.totals.discounts).not.toBe(24.09);
+  });
+
+  it("3. Arithmetic validation rejects a tampered fixture (totals ±$1)", () => {
+    const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
+    const parsed = parseOrderHTML(html);
+    parsed.totals.total_charged = 100.00; // Tampered
+    const isValid = validateOrderTotals(parsed);
+    expect(isValid).toBe(false);
+  });
+
+  it("4. Missing payment line => nulls, not a guess", () => {
+    const html = "<div>Total Charged $15.00</div><div>Subtotal $15.00</div>";
+    const parsed = parseOrderHTML(html);
+    expect(parsed.payment.credits_amount).toBeNull();
+    expect(parsed.payment.card_amount).toBeNull();
+  });
+
+  it("5. Grocery fixture => groceries; assert category != 'dining' (I4)", () => {
+    const html = readFileSync(resolve(fixturesDir, "doordash-credits-grocery.html"), "utf-8");
+    const parsed = parseOrderHTML(html);
+    const resolved = resolveMerchantCategory(parsed.merchant_name);
+    expect(resolved.category).toBe("groceries");
+    expect(resolved.category).not.toBe("dining");
+  });
+
+  it("6. Unresolvable merchant => other + review flag, never dining", () => {
+    const resolved = resolveMerchantCategory("Random Food Truck");
+    expect(resolved.category).toBe("other");
+    expect(resolved.flagReview).toBe(true);
+    expect(resolved.category).not.toBe("dining");
+  });
+
+  it("7. Rating scale accepts only 'again', 'fine', 'never'", () => {
+    const validRatings = ["again", "fine", "never"];
+    const invalidRating = "superb";
+    expect(validRatings.includes("again")).toBe(true);
+    expect(validRatings.includes(invalidRating)).toBe(false);
+  });
+});