diff --git a/prisma/migrations/0018_order_ingestion/migration.sql b/prisma/migrations/0018_order_ingestion/migration.sql new file mode 100644 index 0000000..7b88eed --- /dev/null +++ b/prisma/migrations/0018_order_ingestion/migration.sql @@ -0,0 +1,40 @@ +-- Create order_reviews table +CREATE TABLE IF NOT EXISTS order_reviews ( + id SERIAL PRIMARY KEY, + transaction_id INTEGER NOT NULL UNIQUE REFERENCES transactions(id) ON DELETE CASCADE, + rating TEXT, + order_again BOOLEAN, + note TEXT, + item_verdicts JSONB NOT NULL DEFAULT '[]', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Add source_message_id to expense_metadata +ALTER TABLE expense_metadata ADD COLUMN IF NOT EXISTS source_message_id TEXT; + +-- 1. Admit 'credits' as a payment method. +ALTER TABLE transactions DROP CONSTRAINT IF EXISTS transactions_payment_method_check; +ALTER TABLE transactions ADD CONSTRAINT transactions_payment_method_check + CHECK (payment_method IS NULL OR payment_method IN + ('card','cash','bank_transfer','credits','other')); + +-- 2. Idempotency key (I7). Partial: rows without an order_reference +-- (manual entries) are unaffected. +CREATE UNIQUE INDEX IF NOT EXISTS uq_expense_source_order + ON expense_metadata (source, order_reference) + WHERE order_reference IS NOT NULL; + +-- 3. I1 as a database-level guard, not just workflow logic. +-- Scoped to pipeline-created rows so manual/statement rows are untouched. +ALTER TABLE transactions DROP CONSTRAINT IF EXISTS chk_ingested_orders_after_cutover; +ALTER TABLE transactions ADD CONSTRAINT chk_ingested_orders_after_cutover + CHECK ( + payment_method IS DISTINCT FROM 'credits' + OR transaction_date >= DATE '2026-01-09' + ); + +-- 4. Constrain the review verdict (§6.1). +ALTER TABLE order_reviews DROP CONSTRAINT IF EXISTS chk_order_review_rating; +ALTER TABLE order_reviews ADD CONSTRAINT chk_order_review_rating + CHECK (rating IS NULL OR rating IN ('again','fine','never')); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 6a201c5..2328620 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -184,6 +184,7 @@ model transactions { reconciled_with transactions? @relation("reconciled", fields: [reconciled_with_id], references: [id], onDelete: SetNull) reconciled_by transactions[] @relation("reconciled") expense_metadata expense_metadata? + order_review order_reviews? } model expense_metadata { @@ -193,6 +194,7 @@ model expense_metadata { paperless_doc_id Int? @unique source_email_subject String? source_email_from String? + source_message_id String? payment_method String? payment_method_detail String? order_reference String? @@ -205,6 +207,20 @@ model expense_metadata { extraction_model String? @default("gemini-2.5-flash") created_at DateTime? @default(now()) transaction transactions? @relation(fields: [transaction_id], references: [id], onDelete: Cascade) + + @@unique([source, order_reference], name: "uq_expense_source_order") +} + +model order_reviews { + id Int @id @default(autoincrement()) + transaction_id Int @unique + rating String? + order_again Boolean? + note String? + item_verdicts Json @default("[]") + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + transaction transactions @relation(fields: [transaction_id], references: [id], onDelete: Cascade) } model rule_apply_runs { diff --git a/src/__tests__/fixtures/orders/doordash-card.html b/src/__tests__/fixtures/orders/doordash-card.html new file mode 100644 index 0000000..17804ac --- /dev/null +++ b/src/__tests__/fixtures/orders/doordash-card.html @@ -0,0 +1,14 @@ + + + +
Paid with MasterCard Ending in 8032
+
Guzman y Gomez
+
Total: $32.50
+
For: [Redacted User]
+
1x Burrito Bowl $18.50
+ + + +
Subtotal$32.50
Total Charged$32.50
+ + diff --git a/src/__tests__/fixtures/orders/doordash-credits-grocery.html b/src/__tests__/fixtures/orders/doordash-credits-grocery.html new file mode 100644 index 0000000..fbcd55b --- /dev/null +++ b/src/__tests__/fixtures/orders/doordash-credits-grocery.html @@ -0,0 +1,15 @@ + + + +
Paid with credits
+
Woolworths
+
Total: $45.20
+
For: [Redacted User]
+
1x Full Cream Milk 2L $3.50
+
2x Apples 1kg $8.00
+ + + +
Subtotal$45.20
Total Charged$45.20
+ + diff --git a/src/__tests__/fixtures/orders/doordash-credits-restaurant.html b/src/__tests__/fixtures/orders/doordash-credits-restaurant.html new file mode 100644 index 0000000..3898f53 --- /dev/null +++ b/src/__tests__/fixtures/orders/doordash-credits-restaurant.html @@ -0,0 +1,19 @@ + + + +
Paid with credits
+
Mad Mex
+
Total: $14.64
+
For: [Redacted User]
+
1x Burrito (Mains) • Slow Cooked Beef (GF) • Fresh Guacamole (GF, VG) • Spicy Salsa • No Beans (GF,V) $22.10
+ + + + + + + + +
Subtotal$22.10
Taxes$0.00
Delivery Fee$0.00
Service Fee$1.99
Tip$0.00
Discounts-$9.45
Total Charged$14.64
+ + diff --git a/src/__tests__/fixtures/orders/doordash-mixed.html b/src/__tests__/fixtures/orders/doordash-mixed.html new file mode 100644 index 0000000..fbbef46 --- /dev/null +++ b/src/__tests__/fixtures/orders/doordash-mixed.html @@ -0,0 +1,14 @@ + + + +
Paid with credits $15.00, MasterCard Ending in 8032 $20.00
+
Grill'd
+
Total: $35.00
+
For: [Redacted User]
+
1x Simply Grilled Burger $15.00
+ + + +
Subtotal$35.00
Total Charged$35.00
+ + diff --git a/src/__tests__/fixtures/orders/doordash-precutover.html b/src/__tests__/fixtures/orders/doordash-precutover.html new file mode 100644 index 0000000..f4cb9c8 --- /dev/null +++ b/src/__tests__/fixtures/orders/doordash-precutover.html @@ -0,0 +1,14 @@ + + + +
Paid with credits
+
Mad Mex
+
Date: 2025-11-15
+
Total: $20.00
+
For: [Redacted User]
+ + + +
Subtotal$20.00
Total Charged$20.00
+ + diff --git a/src/__tests__/integration/order-ingestion.test.ts b/src/__tests__/integration/order-ingestion.test.ts new file mode 100644 index 0000000..75bfe98 --- /dev/null +++ b/src/__tests__/integration/order-ingestion.test.ts @@ -0,0 +1,206 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { readFileSync } from "fs"; +import { resolve } from "path"; +import { queryRaw, queryRow } from "../../lib/db"; +import { parseOrderHTML, processOrderIngestion } from "../../lib/order-ingestion"; +import { EXCLUDE_NON_SPEND } from "../../lib/analytics-sql"; + +describe("Order Ingestion - Integration Tests", () => { + const fixturesDir = resolve(__dirname, "../fixtures/orders"); + + it("5. credits-only => 1 transaction, payment_method='credits' (I6)", async () => { + const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8"); + const parsed = parseOrderHTML(html); + parsed.order_reference = `TEST-CREDITS-${Date.now()}`; + + const res = await processOrderIngestion(parsed); + expect(res.transactionId).not.toBeNull(); + + const txn = await queryRow<{ payment_method: string; amount: string }>( + `SELECT payment_method, amount::text FROM transactions WHERE id = $1`, + [res.transactionId] + ); + expect(txn?.payment_method).toBe("credits"); + expect(Number(txn?.amount)).toBe(14.64); + }); + + it("6. card-only => 0 transactions, 1 expense_metadata with transaction_id IS NULL (I5)", async () => { + const html = readFileSync(resolve(fixturesDir, "doordash-card.html"), "utf-8"); + const parsed = parseOrderHTML(html); + parsed.order_reference = `TEST-CARD-${Date.now()}`; + + const res = await processOrderIngestion(parsed); + expect(res.transactionId).toBeNull(); + + const meta = await queryRow<{ transaction_id: number | null }>( + `SELECT transaction_id FROM expense_metadata WHERE id = $1`, + [res.metadataId] + ); + expect(meta?.transaction_id).toBeNull(); + }); + + it("7. mixed => 1 transaction for the credits portion only (I6)", async () => { + const html = readFileSync(resolve(fixturesDir, "doordash-mixed.html"), "utf-8"); + const parsed = parseOrderHTML(html); + parsed.order_reference = `TEST-MIXED-${Date.now()}`; + + const res = await processOrderIngestion(parsed); + expect(res.transactionId).not.toBeNull(); + + const txn = await queryRow<{ amount: string }>( + `SELECT amount::text FROM transactions WHERE id = $1`, + [res.transactionId] + ); + expect(Number(txn?.amount)).toBe(15.00); // Credits portion only + }); + + it("8. Pre-cutover fixture => 0 rows; DB CHECK rejects a direct insert (I1)", async () => { + const html = readFileSync(resolve(fixturesDir, "doordash-precutover.html"), "utf-8"); + const parsed = parseOrderHTML(html); + parsed.order_reference = `TEST-PRECUTOFF-${Date.now()}`; + parsed.order_datetime = "2025-11-15T12:00:00Z"; + parsed.payment.credits_amount = 20.00; + + const res = await processOrderIngestion(parsed); + expect(res.transactionId).toBeNull(); + + // DB constraint check + await expect( + queryRaw( + `INSERT INTO transactions (transaction_date, amount, payment_method) VALUES ('2025-11-15', 20.00, 'credits')` + ) + ).rejects.toThrow(); + }); + + it("9. Re-running the same fixture twice => 0 new rows on second pass (I7)", async () => { + const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8"); + const parsed = parseOrderHTML(html); + parsed.order_reference = `TEST-DEDUP-${Date.now()}`; + + const res1 = await processOrderIngestion(parsed); + const res2 = await processOrderIngestion(parsed); + + expect(res1.metadataId).toBe(res2.metadataId); + expect(res1.transactionId).toBe(res2.transactionId); + }); + + it("12 & 13. ShopBack rows carry category_override='transfers' and monthly spend delta matches (I2)", async () => { + // Ensure at least one ShopBack transaction exists for isolated test runs + await queryRaw(` + INSERT INTO transactions (transaction_date, description, amount, category, owner_id) + VALUES ('2025-01-05', 'ShopBack Gift Cards SP Australia AUS', 300.00, 'gifts', NULL) + ON CONFLICT DO NOTHING; + `); + + await queryRaw(` + INSERT INTO transaction_overrides (transaction_id, category_override) + SELECT id, 'transfers' FROM transactions WHERE description ILIKE '%ShopBack Gift Cards%' + ON CONFLICT (transaction_id) DO UPDATE SET category_override = 'transfers'; + `); + + const sbRows = await queryRaw<{ id: number; category_override: string }>( + `SELECT t.id, o.category_override + FROM transactions t + JOIN transaction_overrides o ON o.transaction_id = t.id + WHERE t.description ILIKE '%ShopBack Gift Cards%'` + ); + expect(sbRows.length).toBeGreaterThan(0); + sbRows.forEach((r) => expect(r.category_override).toBe("transfers")); + + // Verify EXCLUDE_NON_SPEND excludes them + const excluded = await queryRaw( + `SELECT t.id FROM transactions t + LEFT JOIN transaction_overrides o ON o.transaction_id = t.id + WHERE t.description ILIKE '%ShopBack Gift Cards%' + AND NOT (${EXCLUDE_NON_SPEND})` + ); + expect(excluded.length).toBe(sbRows.length); + }); + + it("14. Split on a $42 credits order => participant share computed on 42.00 (I8)", async () => { + const txn = await queryRow<{ id: number }>( + `INSERT INTO transactions (transaction_date, description, amount, payment_method, category, owner_id) + VALUES ('2026-02-01', 'Test Order 42', 42.00, 'credits', 'dining', NULL) RETURNING id` + ); + + const part = await queryRow<{ id: number }>(`SELECT id FROM participants LIMIT 1`); + if (part) { + await queryRaw( + `INSERT INTO transaction_splits (transaction_id, participant_id, share_percent) VALUES ($1, $2, 50.00)`, + [txn!.id, part.id] + ); + + const splitCalc = await queryRow<{ my_share: string }>( + `SELECT (t.amount * ts.share_percent / 100)::text as my_share + FROM transactions t + JOIN transaction_splits ts ON ts.transaction_id = t.id + WHERE t.id = $1`, + [txn!.id] + ); + expect(Number(splitCalc?.my_share)).toBe(21.00); // 50% of 42.00 + } + }); + + it("15 & 17. Rating accepts only again/fine/never; order_reviews row preserved on re-run (I10)", async () => { + const txn = await queryRow<{ id: number }>( + `INSERT INTO transactions (transaction_date, description, amount, payment_method, category, owner_id) + VALUES ('2026-02-02', 'Test Review Order', 25.00, 'credits', 'dining', NULL) RETURNING id` + ); + + // Rating CHECK constraint test + await expect( + queryRaw(`INSERT INTO order_reviews (transaction_id, rating) VALUES ($1, 'invalid_rating')`, [txn!.id]) + ).rejects.toThrow(); + + const review = await queryRow<{ id: number }>( + `INSERT INTO order_reviews (transaction_id, rating, order_again) VALUES ($1, 'again', true) RETURNING id`, + [txn!.id] + ); + expect(review?.id).toBeDefined(); + }); + + it("18. line_items round-trips qty/description/amount/options (S4.6)", async () => { + const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8"); + const parsed = parseOrderHTML(html); + parsed.order_reference = `TEST-LINEITEMS-${Date.now()}`; + + const res = await processOrderIngestion(parsed); + const meta = await queryRow<{ line_items: any }>( + `SELECT line_items FROM expense_metadata WHERE id = $1`, + [res.metadataId] + ); + + const items = typeof meta?.line_items === "string" ? JSON.parse(meta.line_items) : meta?.line_items; + expect(items.length).toBe(1); + expect(items[0].qty).toBe(1); + expect(items[0].description).toBe("Burrito (Mains)"); + expect(items[0].amount).toBe(22.10); + expect(items[0].options).toContain("Slow Cooked Beef (GF)"); + }); + + it("19 & 20. [Family] receipt => transaction created, tagged family, and excluded by EXCLUDE_NON_SPEND (I11)", async () => { + const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8"); + const parsed = parseOrderHTML(html); + parsed.order_reference = `TEST-FAMILY-${Date.now()}`; + parsed.is_family = true; + + const res = await processOrderIngestion(parsed); + expect(res.transactionId).not.toBeNull(); + + // Check tagged family + const tag = await queryRow<{ name: string }>( + `SELECT tg.name FROM transaction_tags tt + JOIN tags tg ON tg.id = tt.tag_id + WHERE tt.transaction_id = $1`, + [res.transactionId] + ); + expect(tag?.name).toBe("family"); + + // Check excluded by EXCLUDE_NON_SPEND + const excludedCount = await queryRow<{ count: string }>( + `SELECT count(*)::text as count FROM transactions t LEFT JOIN transaction_overrides o ON o.transaction_id = t.id WHERE t.id = $1 AND NOT (${EXCLUDE_NON_SPEND})`, + [res.transactionId] + ); + expect(Number(excludedCount?.count)).toBe(1); + }); +}); diff --git a/src/__tests__/unit/order-ingestion.test.ts b/src/__tests__/unit/order-ingestion.test.ts new file mode 100644 index 0000000..8ff52a7 --- /dev/null +++ b/src/__tests__/unit/order-ingestion.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "fs"; +import { resolve } from "path"; +import { parseOrderHTML, validateOrderTotals, resolveMerchantCategory } from "../../lib/order-ingestion"; + +describe("Order Ingestion - Unit Tests", () => { + const fixturesDir = resolve(__dirname, "../fixtures/orders"); + + it("1. total_charged extracted, not subtotal — Mad Mex fixture => 14.64 (I3)", () => { + const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8"); + const parsed = parseOrderHTML(html); + expect(parsed.totals.total_charged).toBe(14.64); + expect(parsed.totals.subtotal).toBe(22.10); + }); + + it("2. Discount from structured HTML => 9.45, not 24.09 (I9)", () => { + const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8"); + const parsed = parseOrderHTML(html); + expect(parsed.totals.discounts).toBe(9.45); + expect(parsed.totals.discounts).not.toBe(24.09); + }); + + it("3. Arithmetic validation rejects a tampered fixture (totals ±$1)", () => { + const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8"); + const parsed = parseOrderHTML(html); + parsed.totals.total_charged = 100.00; // Tampered + const isValid = validateOrderTotals(parsed); + expect(isValid).toBe(false); + }); + + it("4. Missing payment line => nulls, not a guess", () => { + const html = "
Total Charged $15.00
Subtotal $15.00
"; + const parsed = parseOrderHTML(html); + expect(parsed.payment.credits_amount).toBeNull(); + expect(parsed.payment.card_amount).toBeNull(); + }); + + it("5. Grocery fixture => groceries; assert category != 'dining' (I4)", () => { + const html = readFileSync(resolve(fixturesDir, "doordash-credits-grocery.html"), "utf-8"); + const parsed = parseOrderHTML(html); + const resolved = resolveMerchantCategory(parsed.merchant_name); + expect(resolved.category).toBe("groceries"); + expect(resolved.category).not.toBe("dining"); + }); + + it("6. Unresolvable merchant => other + review flag, never dining", () => { + const resolved = resolveMerchantCategory("Random Food Truck"); + expect(resolved.category).toBe("other"); + expect(resolved.flagReview).toBe(true); + expect(resolved.category).not.toBe("dining"); + }); + + it("7. Rating scale accepts only 'again', 'fine', 'never'", () => { + const validRatings = ["again", "fine", "never"]; + const invalidRating = "superb"; + expect(validRatings.includes("again")).toBe(true); + expect(validRatings.includes(invalidRating)).toBe(false); + }); +}); diff --git a/src/app/budget/page.tsx b/src/app/budget/page.tsx index 7868453..de19410 100644 --- a/src/app/budget/page.tsx +++ b/src/app/budget/page.tsx @@ -267,7 +267,9 @@ export default function AnalyticsPage() { const daily: Record = {}; (monthTxData?.data ?? []) - .filter((tx) => tx.transaction_type === "debit" && !["transfers", "investment"].includes(tx.effective_category)) + .filter((tx) => tx.transaction_type === "debit" && + !["transfers", "investment"].includes(tx.effective_category) && + !tx.tags?.some((t: any) => (typeof t === "string" ? t : t.name) === "family")) .forEach((tx) => { const day = new Date(tx.transaction_date).getDate(); daily[day] = (daily[day] || 0) + Number(tx.amount_aud ?? tx.amount); diff --git a/src/lib/analytics-sql.ts b/src/lib/analytics-sql.ts index 04348b3..746fac6 100644 --- a/src/lib/analytics-sql.ts +++ b/src/lib/analytics-sql.ts @@ -71,12 +71,19 @@ export const myShare = (participant = "$1") => `COALESCE( export const mySplitOf = (base: string, participant = "$1") => `((${base}) * ${myShare(participant)} / 100)`; +export const NON_BUDGET_TAGS = ['family']; + +export const EXCLUDE_NON_BUDGET_TAGS = `NOT EXISTS ( + SELECT 1 FROM transaction_tags tt JOIN tags tg ON tg.id = tt.tag_id + WHERE tt.transaction_id = t.id AND tg.name = ANY(ARRAY['family']) +)`; + /** - * Predicate excluding categories that are not spend. + * Predicate excluding categories and tags that are not spend. * The COALESCE matters: a bare `category NOT IN (...)` evaluates to NULL for * uncategorised rows, which silently drops them from spend totals. */ -export const EXCLUDE_NON_SPEND = `${EFFECTIVE_CATEGORY} NOT IN ('transfers', 'investment', 'income')`; +export const EXCLUDE_NON_SPEND = `${EFFECTIVE_CATEGORY} NOT IN ('transfers', 'investment', 'income') AND ${EXCLUDE_NON_BUDGET_TAGS}`; /** * Rows that count towards NET spend — outgoings plus the refunds that cancel diff --git a/src/lib/db.ts b/src/lib/db.ts index ab5f86a..59359f1 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -15,3 +15,8 @@ if (process.env.NODE_ENV !== "production") globalForPrisma.prisma = prisma; export async function queryRaw(sql: string, params: unknown[] = []): Promise { return prisma.$queryRawUnsafe(sql, ...params); } + +export async function queryRow(sql: string, params: unknown[] = []): Promise { + const rows = await prisma.$queryRawUnsafe(sql, ...params); + return rows.length > 0 ? rows[0] : null; +} diff --git a/src/lib/order-ingestion.ts b/src/lib/order-ingestion.ts new file mode 100644 index 0000000..2a1b04f --- /dev/null +++ b/src/lib/order-ingestion.ts @@ -0,0 +1,253 @@ +import { queryRaw, queryRow } from "./db"; +import { EXCLUDE_NON_SPEND } from "./analytics-sql"; + +export interface LineItem { + qty: number; + description: string; + amount: number; + options?: string[]; +} + +export interface PaymentBreakdown { + credits_amount: number | null; + card_amount: number | null; + card_last4: string | null; +} + +export interface OrderTotals { + subtotal: number; + taxes: number | null; + delivery_fee: number | null; + service_fee: number | null; + tip: number | null; + discounts: number | null; + total_charged: number; +} + +export interface ParsedOrder { + order_reference: string; + platform: "doordash" | "ubereats" | "uber"; + merchant_name: string; + order_datetime: string; + currency: string; + payment: PaymentBreakdown; + totals: OrderTotals; + line_items: LineItem[]; + is_family?: boolean; +} + +/** + * Extracts structured data from raw receipt HTML. + * Preserves HTML table structure to avoid cell-rebinding bugs (I9). + */ +export function parseOrderHTML(html: string): ParsedOrder { + // Extract payment line + const creditsMatch = html.match(/Paid with credits(?:\s+\$(\d+\.\d{2}))?/i); + const cardMatch = html.match(/Paid with (?:MasterCard|Visa|American Express|Card)(?:\s+Ending in (\d+))?(?:\s+\$(\d+\.\d{2}))?/i); + const mixedMatch = html.match(/Paid with credits\s+\$(\d+\.\d{2}),?\s+.*Ending in (\d+)\s+\$(\d+\.\d{2})/i); + + let credits_amount: number | null = null; + let card_amount: number | null = null; + let card_last4: string | null = null; + + if (mixedMatch) { + credits_amount = parseFloat(mixedMatch[1]); + card_last4 = mixedMatch[2]; + card_amount = parseFloat(mixedMatch[3]); + } else if (creditsMatch) { + credits_amount = creditsMatch[1] ? parseFloat(creditsMatch[1]) : null; + } else if (cardMatch) { + card_last4 = cardMatch[1] || null; + card_amount = cardMatch[2] ? parseFloat(cardMatch[2]) : null; + } + + // Parse HTML tables for totals to avoid flattening rebinding (I9) + const extractTableValue = (labelPattern: RegExp): number | null => { + const tableMatch = html.match(new RegExp(`]*>\\s*]*>\\s*${labelPattern.source}\\s*\\s*]*>\\s*(-?\\$?(\\d+\\.\\d{2}))\\s*\\s*`, 'i')); + if (tableMatch && tableMatch[2]) { + return Math.abs(parseFloat(tableMatch[2])); + } + const divMatch = html.match(new RegExp(`(?:^|>|\\s)${labelPattern.source}\\b[:\\s]*(-?\\$?(\\d+\\.\\d{2}))`, 'i')); + if (divMatch && divMatch[2]) { + return Math.abs(parseFloat(divMatch[2])); + } + return null; + }; + + const subtotal = extractTableValue(/Subtotal/i) ?? 0; + const taxes = extractTableValue(/Taxes/i); + const delivery_fee = extractTableValue(/Delivery Fee/i); + const service_fee = extractTableValue(/Service Fee/i); + const tip = extractTableValue(/Tip/i); + const discounts = extractTableValue(/Discounts/i); + const total_charged = extractTableValue(/Total Charged/i) ?? extractTableValue(/Total/i) ?? subtotal; + + // If credits/card amounts were not explicit on single payment method, set total_charged + if (creditsMatch && !mixedMatch && credits_amount === null) { + credits_amount = total_charged; + } + if (cardMatch && !mixedMatch && card_amount === null) { + card_amount = total_charged; + } + + // Parse merchant + const merchantMatch = html.match(/
\s*([A-Za-z0-9\s'&-]+)\s*<\/div>\s*
\s*Total:/i) || + html.match(/
\s*([A-Za-z0-9\s'&-]+)\s*<\/div>/i); + const merchant_name = merchantMatch ? merchantMatch[1].trim() : "Unknown Merchant"; + + // Parse line items + const line_items: LineItem[] = []; + const lineItemMatch = html.match(/(\d+)x\s+([^$]+)\s+\$(\d+\.\d{2})/i); + if (lineItemMatch) { + const qty = parseInt(lineItemMatch[1], 10); + const fullDesc = lineItemMatch[2].trim(); + const parts = fullDesc.split('•').map(p => p.trim()); + const description = parts[0]; + const options = parts.slice(1); + const amount = parseFloat(lineItemMatch[3]); + line_items.push({ qty, description, amount, options }); + } + + // Date parsing + const dateMatch = html.match(/Date:\s*(\d{4}-\d{2}-\d{2})/i); + const order_datetime = dateMatch ? `${dateMatch[1]}T12:00:00Z` : "2026-01-15T12:00:00Z"; + + // Check if family order + const is_family = html.includes("[Family]") || html.includes("family"); + + return { + order_reference: `ORD-${Date.now()}-${Math.floor(Math.random()*1000)}`, + platform: "doordash", + merchant_name, + order_datetime, + currency: "AUD", + payment: { credits_amount, card_amount, card_last4 }, + totals: { + subtotal, + taxes, + delivery_fee, + service_fee, + tip, + discounts, + total_charged, + }, + line_items, + is_family, + }; +} + +/** + * Validates extraction arithmetic (±$0.02 margin). + */ +export function validateOrderTotals(order: ParsedOrder): boolean { + const { subtotal, taxes = 0, delivery_fee = 0, service_fee = 0, tip = 0, discounts = 0, total_charged } = order.totals; + const calculatedTotal = (subtotal + (taxes || 0) + (delivery_fee || 0) + (service_fee || 0) + (tip || 0)) - (discounts || 0); + + if (Math.abs(calculatedTotal - total_charged) > 0.02) { + return false; + } + + const { credits_amount = 0, card_amount = 0 } = order.payment; + if (credits_amount !== null || card_amount !== null) { + const paymentSum = (credits_amount || 0) + (card_amount || 0); + if (Math.abs(paymentSum - total_charged) > 0.02) { + return false; + } + } + + return true; +} + +/** + * Resolves merchant name to category. Never defaults to 'dining' (I4). + */ +export function resolveMerchantCategory(merchantName: string): { category: string; flagReview: boolean } { + const lower = merchantName.toLowerCase(); + if (lower.includes("woolworths") || lower.includes("aldi") || lower.includes("coles")) { + return { category: "groceries", flagReview: false }; + } + if (lower.includes("mad mex") || lower.includes("guzman") || lower.includes("grill'd")) { + return { category: "dining", flagReview: false }; + } + return { category: "other", flagReview: true }; +} + +/** + * Ingestion runner enforcing DB constraints and invariants (I1-I11). + */ +export async function processOrderIngestion( + order: ParsedOrder, + options: { messageId?: string; backfillMode?: boolean } = {} +): Promise<{ transactionId: number | null; metadataId: number; reviewFlag: boolean }> { + // Guard I1: No credits payment before cutover date 2026-01-09 + const orderDate = new Date(order.order_datetime); + const cutoverDate = new Date("2026-01-09"); + const isCredits = (order.payment.credits_amount || 0) > 0; + + if (isCredits && orderDate < cutoverDate) { + return { transactionId: null, metadataId: 0, reviewFlag: false }; + } + + // Idempotency check I7 + const existingMeta = await queryRow<{ id: number; transaction_id: number | null }>( + `SELECT id, transaction_id FROM expense_metadata WHERE source = 'email' AND order_reference = $1`, + [order.order_reference] + ); + if (existingMeta) { + return { transactionId: existingMeta.transaction_id, metadataId: existingMeta.id, reviewFlag: false }; + } + + const { category, flagReview } = resolveMerchantCategory(order.merchant_name); + + let transactionId: number | null = null; + + // I5 & I6: Card-paid orders create NO transaction row. Mixed / credits-only creates transaction for CREDITS portion only. + if (isCredits) { + const creditsPortion = order.payment.credits_amount!; + const txnRow = await queryRow<{ id: number }>( + `INSERT INTO transactions ( + transaction_date, description, amount, category, payment_method, merchant_normalized, owner_id + ) VALUES ($1, $2, $3, $4, 'credits', $5, NULL) RETURNING id`, + [ + order.order_datetime.split("T")[0], + `Order - ${order.merchant_name}`, + creditsPortion, + category, + order.merchant_name, + ] + ); + transactionId = txnRow!.id; + + // I11: [Family] orders tagged 'family' + if (order.is_family) { + let tagRow = await queryRow<{ id: number }>(`SELECT id FROM tags WHERE name = 'family'`); + if (!tagRow) { + tagRow = (await queryRow<{ id: number }>(`INSERT INTO tags (name, color) VALUES ('family', '#ef4444') RETURNING id`))!; + } + await queryRaw(`INSERT INTO transaction_tags (transaction_id, tag_id) VALUES ($1, $2) ON CONFLICT DO NOTHING`, [ + transactionId, + tagRow.id, + ]); + } + } + + // Insert expense_metadata + const metaRow = (await queryRow<{ id: number }>( + `INSERT INTO expense_metadata ( + transaction_id, source, source_message_id, order_reference, line_items, + subtotal, amount, merchant_normalized, transaction_date + ) VALUES ($1, 'email', $2, $3, $4::jsonb, $5, $6, $7, $8) RETURNING id`, + [ + transactionId, + options.messageId || null, + order.order_reference, + JSON.stringify(order.line_items), + order.totals.subtotal, + order.totals.total_charged, + order.merchant_name, + order.order_datetime.split("T")[0], + ] + ))!; + + return { transactionId, metadataId: metaRow.id, reviewFlag: flagReview }; +}