finance-app/src/__tests__/integration/order-ingestion.test.ts

import { describe, it, expect, beforeEach } from "vitest";
import { readFileSync } from "fs";
import { resolve } from "path";
import { queryRaw, queryRow } from "../../lib/db";
import {
  parseOrderHTML,
  validateOrderTotals,
  processOrderIngestion,
  reconcilePendingOrders,
  parseOrderAmendment,
  applyOrderAmendment,
  NotAReceiptError,
  type MessageMeta,
} from "../../lib/order-ingestion";
import { EXCLUDE_NON_SPEND } from "../../lib/analytics-sql";
import { bankLabel, needsCardMatch } from "../../lib/queries";

/**
 * These run against REAL captured receipts, not synthetic fixtures. The earlier
 * suite passed 31/31 against fixtures written to satisfy the parser, while the
 * parser could not read a single real email. Fixtures live in
 * __tests__/fixtures/orders/real/ and are unmodified message bodies.
 */
const dir = resolve(__dirname, "../fixtures/orders/real");
const html = (f: string) => readFileSync(resolve(dir, `${f}.html`), "utf-8");

const meta = (over: Partial<MessageMeta> = {}): MessageMeta => ({
  messageId: `test-${Math.random().toString(36).slice(2)}`,
  subject: "Order Confirmation for Siddharth from Mad Mex",
  receivedAt: "2026-07-16T03:34:00Z",
  sender: "DoorDash Order <no-reply@doordash.com>",
  ...over,
});

describe("Order parsing — real receipts", () => {
  it("reads DoorDash totals structurally, not by flattening (I9)", () => {
    const p = parseOrderHTML(html("dd-01"), meta());
    expect(p.merchant_name).toBe("Mad Mex");
    expect(p.totals.total_charged).toBe(14.64);
    expect(p.payment.credits_amount).toBe(14.64);
    expect(p.line_items).toHaveLength(1);
    expect(p.line_items[0].description).toBe("Burrito (Mains)");
    expect(p.line_items[0].options).toContain("Slow Cooked Beef (GF)");
  });

  it("does NOT gate on DoorDash's fee breakdown, which genuinely does not reconcile", () => {
    // Real receipt: subtotal 22.10 + service 1.99 - Discounts 24.09 = 0.00,
    // against a stated total of 14.64. DoorDash prints this; it is not a parse
    // artefact. Recorded here so nobody "fixes" the parser to force it to sum.
    const p = parseOrderHTML(html("dd-01"), meta());
    expect(p.totals.subtotal).toBe(22.10);
    expect(p.totals.discounts).toBe(24.09);
    expect(validateOrderTotals(p, html("dd-01")).ok).toBe(true);
  });

  it("derives order_reference from the message, never randomly (I7)", () => {
    const m = meta({ messageId: "abc123" });
    const a = parseOrderHTML(html("dd-01"), m);
    const b = parseOrderHTML(html("dd-01"), m);
    expect(a.order_reference).toBe(b.order_reference);
    expect(a.order_reference).toBe("msg:abc123");
  });

  it("uses Uber's embedded order UUID as the reference", () => {
    const p = parseOrderHTML(
      html("ue-00"),
      meta({ subject: "Your Wednesday afternoon order with Uber Eats", sender: "uber.com" })
    );
    expect(p.order_reference).toMatch(/^[0-9a-f-]{36}$/);
    expect(p.platform).toBe("ubereats");
  });

  it("takes the order date from the message, not a body string", () => {
    const p = parseOrderHTML(html("dd-01"), meta({ receivedAt: "2026-07-16T03:34:00Z" }));
    expect(p.order_datetime.slice(0, 10)).toBe("2026-07-16");
  });

  it("detects [Family] from the subject prefix, not a body substring (I11)", () => {
    const fam = parseOrderHTML(
      html("ue-04"),
      meta({ subject: "[Family] Your Tuesday evening order with Uber Eats", sender: "uber.com" })
    );
    expect(fam.is_family).toBe(true);
    const notFam = parseOrderHTML(html("dd-01"), meta());
    expect(notFam.is_family).toBe(false);
  });

  it("reads [Family] orders as LKR, not dollars", () => {
    const p = parseOrderHTML(
      html("ue-04"),
      meta({ subject: "[Family] Your Tuesday evening order with Uber Eats", sender: "uber.com" })
    );
    expect(p.currency).toBe("LKR");
    expect(p.totals.total_charged).toBeCloseTo(3783.20, 2);
  });

  it("reads Swiss orders as CHF", () => {
    const p = parseOrderHTML(
      html("ue-26"),
      meta({ subject: "Your Tuesday evening order with Uber Eats", sender: "uber.com" })
    );
    expect(p.currency).toBe("CHF");
    expect(p.totals.total_charged).toBeCloseTo(51.23, 2);
  });

  it("skips a failed payment attempt and takes the successful one", () => {
    // ue-09: "Visa ••••8841 LKR 4,267.01 ... Failed" then "LKR 3,757.01".
    const p = parseOrderHTML(
      html("ue-09"),
      meta({ subject: "[Family] Your Sunday evening order with Uber Eats", sender: "uber.com" })
    );
    expect(p.totals.total_charged).toBeCloseTo(3757.01, 2);
    expect(validateOrderTotals(p, html("ue-09")).ok).toBe(true);
  });

  it("rejects an order-adjustment notice rather than booking $0.00", () => {
    // NotAReceiptError, not OrderParseError: this is expected traffic, so it
    // must be skipped silently. Only a receipt that fails to parse should
    // alert — see the ingest API's error taxonomy.
    expect(() =>
      parseOrderHTML(html("dd-08"), meta({ subject: "Order Confirmation for Siddharth from ALDI" }))
    ).toThrow(NotAReceiptError);
  });

  it("rejects a refund notice rather than inserting a duplicate order", () => {
    expect(() =>
      parseOrderHTML(html("ue-05"), meta({ subject: "Your Tuesday evening order with Uber Eats", sender: "uber.com" }))
    ).toThrow(/refund/i);
  });

  it("reads a grocery Final receipt that has no Total Charged row", () => {
    const p = parseOrderHTML(
      html("dd-10"),
      meta({ subject: "Order Confirmation for Siddharth from Woolworths" })
    );
    expect(p.totals.total_charged).toBeCloseTo(60.93, 2);
    expect(p.payment.ambiguous).toBe(true); // "8032 and/or credits"
  });
});

describe("Order ingestion — invariants", () => {
  beforeEach(async () => {
    await queryRaw(`DELETE FROM expense_metadata WHERE source = 'email'`);
    await queryRaw(`DELETE FROM transactions WHERE description LIKE 'Order - %'`);
    // The statement fixtures these tests insert survived into the next run, and
    // reconcileCardLeg matched a leftover charge at ingest time — so an order
    // meant to park "awaiting_card_statement" resolved immediately instead.
    // That is the whole story behind the intermittent failure in "parks an
    // unresolvable split": not a race, just fixtures that were never cleaned.
    // Must happen BEFORE ingest, which is why cleaning up at the end of the
    // test was not enough.
    await queryRaw(
      `DELETE FROM statements WHERE filename IN ('test-westpac-2026-03.pdf', 'panel-cba.pdf', 'panel-plain.pdf')`
    );
    await queryRaw(
      `DELETE FROM transactions WHERE description IN ('DD *DOORDASH WOOLWORTHS MELBOURNE AUS', 'UBER *EATS ZURICH')`
    );
  });

  it("I6: a credits order creates one transaction at face value", async () => {
    const p = parseOrderHTML(html("dd-01"), meta());
    const res = await processOrderIngestion(p);
    expect(res.transactionId).not.toBeNull();
    const txn = await queryRow<{ amount: string; payment_method: string; category: string }>(
      `SELECT amount::text, payment_method, category FROM transactions WHERE id = $1`,
      [res.transactionId]
    );
    expect(Number(txn!.amount)).toBe(14.64);
    expect(txn!.payment_method).toBe("credits");
    expect(txn!.category).toBe("dining");
  });

  it("I7: re-ingesting the same receipt creates nothing new", async () => {
    const p = parseOrderHTML(html("dd-01"), meta({ messageId: "dedupe-1" }));
    const a = await processOrderIngestion(p);
    const b = await processOrderIngestion(parseOrderHTML(html("dd-01"), meta({ messageId: "dedupe-1" })));
    expect(b.skipped).toBe("already_ingested");
    expect(b.metadataId).toBe(a.metadataId);
    const n = await queryRow<{ c: string }>(
      `SELECT count(*)::text c FROM transactions WHERE description = 'Order - Mad Mex (DoorDash)'`
    );
    expect(Number(n!.c)).toBe(1);
  });

  it("I1: a credits order before the cutover is refused", async () => {
    const p = parseOrderHTML(html("dd-01"), meta({ receivedAt: "2025-11-15T12:00:00Z" }));
    const res = await processOrderIngestion(p);
    expect(res.skipped).toBe("pre_cutover");
    expect(res.transactionId).toBeNull();
    await expect(
      queryRaw(
        `INSERT INTO transactions (transaction_date, amount, payment_method) VALUES ('2025-11-15', 20.00, 'credits')`
      )
    ).rejects.toThrow();
  });

  it("I11: a [Family] order records provenance and creates no transaction", async () => {
    // Requirement was "import them but tag so they're excluded from budgets".
    // Correct mechanism: the CARD statement line is the transaction and carries
    // the family tag. Creating a second, credits-flavoured row duplicated it.
    const p = parseOrderHTML(
      html("ue-04"),
      meta({ subject: "[Family] Your Tuesday evening order with Uber Eats", sender: "uber.com", receivedAt: "2026-07-07T10:08:00Z" })
    );
    const res = await processOrderIngestion(p);
    expect(res.transactionId).toBeNull();
    expect(res.flags).toContain("family_card_settled_no_transaction");

    const meta_ = await queryRow<{ currency: string }>(
      `SELECT currency FROM expense_metadata WHERE id = $1`, [res.metadataId]
    );
    expect(meta_!.currency).toBe("LKR");
  });

  it("a foreign-currency order records the original amount and code", async () => {
    const p = parseOrderHTML(
      html("ue-26"),
      meta({ subject: "Your Tuesday evening order with Uber Eats", sender: "uber.com", receivedAt: "2026-04-07T08:53:00Z" })
    );
    // Card-settled Swiss order: no credits leg, so no transaction (I5).
    const res = await processOrderIngestion(p);
    expect(res.transactionId).toBeNull();
    const meta_ = await queryRow<{ currency: string }>(
      `SELECT currency FROM expense_metadata WHERE id = $1`,
      [res.metadataId]
    );
    expect(meta_!.currency).toBe("CHF");
  });

  it("parks an unresolvable split instead of guessing, then resolves it once the statement lands", async () => {
    const p = parseOrderHTML(
      html("dd-10"),
      meta({ subject: "Order Confirmation for Siddharth from Woolworths", receivedAt: "2026-03-02T12:00:00Z" })
    );
    const res = await processOrderIngestion(p);
    expect(res.transactionId).toBeNull();
    expect(res.flags).toContain("awaiting_card_statement");

    // Statement arrives: card 8032 took 40.93 of the 60.93 order.
    const st = await queryRow<{ id: number }>(
      `INSERT INTO statements (bank_name, account_number, filename)
       VALUES ('Westpac','5163103015778032','test-westpac-2026-03.pdf') RETURNING id`
    );
    await queryRaw(
      `INSERT INTO transactions (statement_id, transaction_date, description, amount, transaction_type)
       VALUES ($1, '2026-03-02', 'DD *DOORDASH WOOLWORTHS MELBOURNE AUS', 40.93, 'debit')`,
      [st!.id]
    );

    const out = await reconcilePendingOrders();
    // Deliberately not asserting global counts: reconcilePendingOrders() scans
    // every pending row in the database, so another test's leftovers change the
    // totals. Assert on THIS order's outcome instead — that is what the test is
    // actually about, and it does not depend on what else is in the table.
    expect(out.examined).toBeGreaterThanOrEqual(1);

    const credits = await queryRow<{ amount: string }>(
      `SELECT t.amount::text FROM transactions t
        JOIN expense_metadata em ON em.transaction_id = t.id
       WHERE em.id = $1`,
      [res.metadataId]
    );
    expect(Number(credits!.amount)).toBeCloseTo(20.00, 2); // 60.93 - 40.93
  });

  it("reconciliation is idempotent — a second pass creates nothing", async () => {
    // A first pass has already run above; a second must add nothing. Scoped to
    // 'Order - %' rows so unrelated fixtures cannot move the number.
    const q = `SELECT count(*)::text c FROM transactions WHERE description LIKE 'Order - %'`;
    const before = await queryRow<{ c: string }>(q);
    const out = await reconcilePendingOrders();
    const after = await queryRow<{ c: string }>(q);
    expect(out.created).toBe(0);
    expect(after!.c).toBe(before!.c);
  });

  it("EXCLUDE_NON_SPEND removes family-tagged rows", async () => {
    const txn = await queryRow<{ id: number }>(
      `INSERT INTO transactions (transaction_date, description, amount, category, transaction_type)
       VALUES ('2026-03-01','Order - Family Test', 50.00, 'dining', 'debit') RETURNING id`
    );
    const tag = await queryRow<{ id: number }>(
      `INSERT INTO tags (name, color) VALUES ('family','#ef4444')
       ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id`
    );
    await queryRaw(`INSERT INTO transaction_tags (transaction_id, tag_id) VALUES ($1,$2)`, [txn!.id, tag!.id]);

    const visible = await queryRaw(
      `SELECT t.id FROM transactions t
        LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
       WHERE t.id = $1 AND (${EXCLUDE_NON_SPEND})`,
      [txn!.id]
    );
    expect(visible).toHaveLength(0);
  });
});

describe("Refund amendments", () => {
  const ueMeta = (over = {}) => meta({
    subject: "Your Tuesday evening order with Uber Eats",
    sender: "uber.com",
    receivedAt: "2026-07-07T08:17:00Z",
    ...over,
  });

  it("parses a refund notice into an amendment, not an order", () => {
    const a = parseOrderAmendment(html("ue-05"), ueMeta());
    expect(a.previous_total).toBeCloseTo(49.94, 2);
    expect(a.refund_amount).toBeCloseTo(4.21, 2);
    expect(a.new_total).toBeCloseTo(45.73, 2);
    expect(a.order_reference).toMatch(/^[0-9a-f-]{36}$/);
  });

  it("reduces the original transaction instead of adding a second row", async () => {
    // Seed the original order this amendment refers to.
    const a = parseOrderAmendment(html("ue-05"), ueMeta());
    const txn = await queryRow<{ id: number }>(
      `INSERT INTO transactions (transaction_date, description, amount, amount_aud, category, payment_method, transaction_type)
       VALUES ('2026-07-07','Order - Coles (Wyndham Vale)', 49.94, 49.94, 'groceries', 'credits', 'debit')
       RETURNING id`
    );
    await queryRaw(
      `INSERT INTO expense_metadata (transaction_id, source, order_reference, amount, transaction_date, flags)
       VALUES ($1,'email',$2, 49.94, '2026-07-07', '[]'::jsonb)`,
      [txn!.id, a.order_reference]
    );

    const before = await queryRow<{ c: string }>(`SELECT count(*)::text c FROM transactions`);
    const res = await applyOrderAmendment(a);
    const after = await queryRow<{ c: string }>(`SELECT count(*)::text c FROM transactions`);

    expect(res.matched).toBe(true);
    expect(after!.c).toBe(before!.c); // amended in place, no second row
    const updated = await queryRow<{ amount: string }>(
      `SELECT amount::text FROM transactions WHERE id = $1`,
      [txn!.id]
    );
    expect(Number(updated!.amount)).toBeCloseTo(45.73, 2);
  });

  it("invents nothing when the original order was never ingested", async () => {
    const a = parseOrderAmendment(html("ue-05"), ueMeta());
    await queryRaw(`DELETE FROM expense_metadata WHERE order_reference = $1`, [a.order_reference]);
    const res = await applyOrderAmendment(a);
    expect(res.matched).toBe(false);
    expect(res.transactionId).toBeNull();
  });
});

describe("[Family] orders are card-settled, not credits", () => {
  it("creates provenance but NO transaction — the statement line is the transaction", async () => {
    // Regression: these were assumed credits-funded because the receipt names
    // the payer and no instrument. The card statement carries all four (CBA
    // ...3893, exact LKR matches), so creating a transaction double-counted
    // spend already recorded.
    const p = parseOrderHTML(
      html("ue-04"),
      meta({ subject: "[Family] Your Tuesday evening order with Uber Eats", sender: "uber.com", receivedAt: "2026-07-07T10:08:00Z" })
    );
    expect(p.flags).toContain("family_card_settled_no_transaction");
    expect(p.payment.credits_amount).toBeNull();

    const res = await processOrderIngestion(p);
    expect(res.transactionId).toBeNull();
    expect(res.metadataId).not.toBeNull();
  });
});

describe("owner scoping", () => {
  it("sets owner_id so the row is visible to the app", async () => {
    // Regression: analytics scope on COALESCE(t.owner_id, s.owner_id). An
    // ingested order has no statement, so a NULL owner_id made all 85
    // backfilled rows invisible in every view while sitting in the table.
    const p = parseOrderHTML(html("dd-01"), meta({ messageId: `owner-${Date.now()}` }));
    const res = await processOrderIngestion(p);
    const row = await queryRow<{ owner_id: number | null }>(
      `SELECT owner_id FROM transactions WHERE id = $1`,
      [res.transactionId]
    );
    expect(row!.owner_id).not.toBeNull();

    const visible = await queryRaw(
      `SELECT t.id FROM transactions t LEFT JOIN statements s ON s.id = t.statement_id
        WHERE t.id = $1 AND COALESCE(t.owner_id, s.owner_id) = $2`,
      [res.transactionId, row!.owner_id]
    );
    expect(visible).toHaveLength(1);
  });
});

describe("how an ingested order presents in the app", () => {
  it("names the platform in the description", async () => {
    // "Order - Burger Corner" gives no way to know where to look for the
    // detail, and the same restaurant can be on both platforms.
    const p = parseOrderHTML(html("dd-01"), meta({ messageId: `desc-${Date.now()}` }));
    const res = await processOrderIngestion(p);
    const row = await queryRow<{ description: string }>(
      `SELECT description FROM transactions WHERE id = $1`, [res.transactionId]
    );
    expect(row!.description).toMatch(/\(DoorDash\)$/);
  });

  it("reads as 'Gift Card', not 'Manual', and stays out of the reconcile queue", async () => {
    // bank_name is derived — no statement means "Manual", which reads as
    // "hand-entered, awaiting a card line". A credits order has no card line
    // coming, ever; 81 of them sat in the queue waiting for one.
    const p = parseOrderHTML(html("dd-01"), meta({ messageId: `bank-${Date.now()}` }));
    const res = await processOrderIngestion(p);

    const row = await queryRow<{ bank_name: string }>(
      `SELECT ${bankLabel()} as bank_name
         FROM transactions t LEFT JOIN statements s ON s.id = t.statement_id
        WHERE t.id = $1`,
      [res.transactionId]
    );
    expect(row!.bank_name).toBe("Gift Card");

    const queued = await queryRaw(
      `SELECT t.id FROM transactions t
        WHERE t.id = $1 AND t.statement_id IS NULL AND ${needsCardMatch("t")}`,
      [res.transactionId]
    );
    expect(queued).toHaveLength(0);
  });

  it("records the platform and the message it came from", async () => {
    const p = parseOrderHTML(
      html("ue-00"),
      meta({ messageId: `prov-${Date.now()}`, subject: "Your Wednesday order with Uber Eats", sender: "uber.com" })
    );
    const res = await processOrderIngestion(p, {
      messageId: `prov-${Date.now()}`,
      subject: "Your Wednesday order with Uber Eats",
      sender: "uber.com",
    });
    const row = await queryRow<{
      platform: string; source_email_from: string; route: { label: string }[];
    }>(
      `SELECT platform, source_email_from, route FROM expense_metadata WHERE id = $1`,
      [res.metadataId]
    );
    expect(row!.platform).toBe("ubereats");
    expect(row!.source_email_from).toBe("uber.com");
    expect(row!.route.map((r) => r.label)).toEqual(["Pick-up", "Delivery"]);
  });
});

describe("receipt lookup for the transaction detail panel", () => {
  // Mirrors /api/transactions/[id]/order — the panel resolves a receipt from
  // either side, and a card-settled order only has the matched_transaction_id
  // side, which is exactly where the detail would otherwise go missing.
  const receiptFor = (txnId: number) =>
    queryRow<{ platform: string; route: { label: string; address: string }[] }>(
      `SELECT platform, route FROM expense_metadata
        WHERE transaction_id = $1 OR matched_transaction_id = $1 LIMIT 1`,
      [txnId]
    );

  it("finds the receipt for a credits order", async () => {
    // ue-00 is Uber Cash — credits, so it creates a transaction. ue-09 names a
    // payer with no instrument and correctly parks awaiting a card statement,
    // which would leave nothing to look the receipt up by.
    const p = parseOrderHTML(
      html("ue-00"),
      meta({ messageId: `panel-${Date.now()}`, subject: "Your Wednesday order with Uber Eats", sender: "uber.com" })
    );
    const res = await processOrderIngestion(p);
    const r = await receiptFor(res.transactionId!);
    expect(r!.platform).toBe("ubereats");
    expect(r!.route.map((x) => x.label)).toEqual(["Pick-up", "Delivery"]);
  });

  it("finds it from the statement line for a card-settled order", async () => {
    const st = await queryRow<{ id: number }>(
      `INSERT INTO statements (bank_name, account_number, filename)
       VALUES ('CBA', '5523504401723893', 'panel-cba.pdf') RETURNING id`
    );
    const card = await queryRow<{ id: number }>(
      `INSERT INTO transactions (statement_id, transaction_date, description, amount, transaction_type)
       VALUES ($1, '2026-04-07', 'UBER *EATS ZURICH', 51.23, 'debit') RETURNING id`,
      [st!.id]
    );
    const m = await queryRow<{ id: number }>(
      `INSERT INTO expense_metadata (source, order_reference, platform, route, matched_transaction_id)
       VALUES ('email', $1, 'ubereats', '[{"label":"Pick-up","time":null,"address":"Ebikon"}]'::jsonb, $2)
       RETURNING id`,
      [`panel-card-${Date.now()}`, card!.id]
    );
    expect(m).not.toBeNull();

    const r = await receiptFor(card!.id);
    expect(r!.platform).toBe("ubereats");
    expect(r!.route[0].address).toBe("Ebikon");
  });

  it("returns nothing for an ordinary transaction", async () => {
    const st = await queryRow<{ id: number }>(
      `INSERT INTO statements (bank_name, account_number, filename)
       VALUES ('CBA', '1111', 'panel-plain.pdf') RETURNING id`
    );
    const t = await queryRow<{ id: number }>(
      `INSERT INTO transactions (statement_id, transaction_date, description, amount, transaction_type)
       VALUES ($1, '2026-04-07', 'COLES 1234', 12.00, 'debit') RETURNING id`,
      [st!.id]
    );
    expect(await receiptFor(t!.id)).toBeNull();
  });
});