feat(orders): implement order ingestion pipeline, review ratings schema, and [Family] exclusion
This commit is contained in:
@@ -0,0 +1,40 @@
|
||||
-- Create order_reviews table
|
||||
CREATE TABLE IF NOT EXISTS order_reviews (
|
||||
id SERIAL PRIMARY KEY,
|
||||
transaction_id INTEGER NOT NULL UNIQUE REFERENCES transactions(id) ON DELETE CASCADE,
|
||||
rating TEXT,
|
||||
order_again BOOLEAN,
|
||||
note TEXT,
|
||||
item_verdicts JSONB NOT NULL DEFAULT '[]',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Add source_message_id to expense_metadata
|
||||
ALTER TABLE expense_metadata ADD COLUMN IF NOT EXISTS source_message_id TEXT;
|
||||
|
||||
-- 1. Admit 'credits' as a payment method.
|
||||
ALTER TABLE transactions DROP CONSTRAINT IF EXISTS transactions_payment_method_check;
|
||||
ALTER TABLE transactions ADD CONSTRAINT transactions_payment_method_check
|
||||
CHECK (payment_method IS NULL OR payment_method IN
|
||||
('card','cash','bank_transfer','credits','other'));
|
||||
|
||||
-- 2. Idempotency key (I7). Partial: rows without an order_reference
|
||||
-- (manual entries) are unaffected.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_expense_source_order
|
||||
ON expense_metadata (source, order_reference)
|
||||
WHERE order_reference IS NOT NULL;
|
||||
|
||||
-- 3. I1 as a database-level guard, not just workflow logic.
|
||||
-- Scoped to pipeline-created rows so manual/statement rows are untouched.
|
||||
ALTER TABLE transactions DROP CONSTRAINT IF EXISTS chk_ingested_orders_after_cutover;
|
||||
ALTER TABLE transactions ADD CONSTRAINT chk_ingested_orders_after_cutover
|
||||
CHECK (
|
||||
payment_method IS DISTINCT FROM 'credits'
|
||||
OR transaction_date >= DATE '2026-01-09'
|
||||
);
|
||||
|
||||
-- 4. Constrain the review verdict (§6.1).
|
||||
ALTER TABLE order_reviews DROP CONSTRAINT IF EXISTS chk_order_review_rating;
|
||||
ALTER TABLE order_reviews ADD CONSTRAINT chk_order_review_rating
|
||||
CHECK (rating IS NULL OR rating IN ('again','fine','never'));
|
||||
@@ -184,6 +184,7 @@ model transactions {
|
||||
reconciled_with transactions? @relation("reconciled", fields: [reconciled_with_id], references: [id], onDelete: SetNull)
|
||||
reconciled_by transactions[] @relation("reconciled")
|
||||
expense_metadata expense_metadata?
|
||||
order_review order_reviews?
|
||||
}
|
||||
|
||||
model expense_metadata {
|
||||
@@ -193,6 +194,7 @@ model expense_metadata {
|
||||
paperless_doc_id Int? @unique
|
||||
source_email_subject String?
|
||||
source_email_from String?
|
||||
source_message_id String?
|
||||
payment_method String?
|
||||
payment_method_detail String?
|
||||
order_reference String?
|
||||
@@ -205,6 +207,20 @@ model expense_metadata {
|
||||
extraction_model String? @default("gemini-2.5-flash")
|
||||
created_at DateTime? @default(now())
|
||||
transaction transactions? @relation(fields: [transaction_id], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([source, order_reference], name: "uq_expense_source_order")
|
||||
}
|
||||
|
||||
model order_reviews {
|
||||
id Int @id @default(autoincrement())
|
||||
transaction_id Int @unique
|
||||
rating String?
|
||||
order_again Boolean?
|
||||
note String?
|
||||
item_verdicts Json @default("[]")
|
||||
created_at DateTime @default(now())
|
||||
updated_at DateTime @updatedAt
|
||||
transaction transactions @relation(fields: [transaction_id], references: [id], onDelete: Cascade)
|
||||
}
|
||||
|
||||
model rule_apply_runs {
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div>Paid with MasterCard Ending in 8032</div>
|
||||
<div>Guzman y Gomez</div>
|
||||
<div>Total: $32.50</div>
|
||||
<div>For: [Redacted User]</div>
|
||||
<div>1x Burrito Bowl $18.50</div>
|
||||
<table>
|
||||
<tr><td>Subtotal</td><td>$32.50</td></tr>
|
||||
<tr><td>Total Charged</td><td>$32.50</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,15 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div>Paid with credits</div>
|
||||
<div>Woolworths</div>
|
||||
<div>Total: $45.20</div>
|
||||
<div>For: [Redacted User]</div>
|
||||
<div>1x Full Cream Milk 2L $3.50</div>
|
||||
<div>2x Apples 1kg $8.00</div>
|
||||
<table>
|
||||
<tr><td>Subtotal</td><td>$45.20</td></tr>
|
||||
<tr><td>Total Charged</td><td>$45.20</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,19 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div>Paid with credits</div>
|
||||
<div>Mad Mex</div>
|
||||
<div>Total: $14.64</div>
|
||||
<div>For: [Redacted User]</div>
|
||||
<div>1x Burrito (Mains) • Slow Cooked Beef (GF) • Fresh Guacamole (GF, VG) • Spicy Salsa • No Beans (GF,V) $22.10</div>
|
||||
<table>
|
||||
<tr><td>Subtotal</td><td>$22.10</td></tr>
|
||||
<tr><td>Taxes</td><td>$0.00</td></tr>
|
||||
<tr><td>Delivery Fee</td><td>$0.00</td></tr>
|
||||
<tr><td>Service Fee</td><td>$1.99</td></tr>
|
||||
<tr><td>Tip</td><td>$0.00</td></tr>
|
||||
<tr><td>Discounts</td><td>-$9.45</td></tr>
|
||||
<tr><td>Total Charged</td><td>$14.64</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,14 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div>Paid with credits $15.00, MasterCard Ending in 8032 $20.00</div>
|
||||
<div>Grill'd</div>
|
||||
<div>Total: $35.00</div>
|
||||
<div>For: [Redacted User]</div>
|
||||
<div>1x Simply Grilled Burger $15.00</div>
|
||||
<table>
|
||||
<tr><td>Subtotal</td><td>$35.00</td></tr>
|
||||
<tr><td>Total Charged</td><td>$35.00</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,14 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div>Paid with credits</div>
|
||||
<div>Mad Mex</div>
|
||||
<div>Date: 2025-11-15</div>
|
||||
<div>Total: $20.00</div>
|
||||
<div>For: [Redacted User]</div>
|
||||
<table>
|
||||
<tr><td>Subtotal</td><td>$20.00</td></tr>
|
||||
<tr><td>Total Charged</td><td>$20.00</td></tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,206 @@
|
||||
import { describe, it, expect, beforeAll } from "vitest";
|
||||
import { readFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
import { queryRaw, queryRow } from "../../lib/db";
|
||||
import { parseOrderHTML, processOrderIngestion } from "../../lib/order-ingestion";
|
||||
import { EXCLUDE_NON_SPEND } from "../../lib/analytics-sql";
|
||||
|
||||
describe("Order Ingestion - Integration Tests", () => {
|
||||
const fixturesDir = resolve(__dirname, "../fixtures/orders");
|
||||
|
||||
it("5. credits-only => 1 transaction, payment_method='credits' (I6)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-CREDITS-${Date.now()}`;
|
||||
|
||||
const res = await processOrderIngestion(parsed);
|
||||
expect(res.transactionId).not.toBeNull();
|
||||
|
||||
const txn = await queryRow<{ payment_method: string; amount: string }>(
|
||||
`SELECT payment_method, amount::text FROM transactions WHERE id = $1`,
|
||||
[res.transactionId]
|
||||
);
|
||||
expect(txn?.payment_method).toBe("credits");
|
||||
expect(Number(txn?.amount)).toBe(14.64);
|
||||
});
|
||||
|
||||
it("6. card-only => 0 transactions, 1 expense_metadata with transaction_id IS NULL (I5)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-card.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-CARD-${Date.now()}`;
|
||||
|
||||
const res = await processOrderIngestion(parsed);
|
||||
expect(res.transactionId).toBeNull();
|
||||
|
||||
const meta = await queryRow<{ transaction_id: number | null }>(
|
||||
`SELECT transaction_id FROM expense_metadata WHERE id = $1`,
|
||||
[res.metadataId]
|
||||
);
|
||||
expect(meta?.transaction_id).toBeNull();
|
||||
});
|
||||
|
||||
it("7. mixed => 1 transaction for the credits portion only (I6)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-mixed.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-MIXED-${Date.now()}`;
|
||||
|
||||
const res = await processOrderIngestion(parsed);
|
||||
expect(res.transactionId).not.toBeNull();
|
||||
|
||||
const txn = await queryRow<{ amount: string }>(
|
||||
`SELECT amount::text FROM transactions WHERE id = $1`,
|
||||
[res.transactionId]
|
||||
);
|
||||
expect(Number(txn?.amount)).toBe(15.00); // Credits portion only
|
||||
});
|
||||
|
||||
it("8. Pre-cutover fixture => 0 rows; DB CHECK rejects a direct insert (I1)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-precutover.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-PRECUTOFF-${Date.now()}`;
|
||||
parsed.order_datetime = "2025-11-15T12:00:00Z";
|
||||
parsed.payment.credits_amount = 20.00;
|
||||
|
||||
const res = await processOrderIngestion(parsed);
|
||||
expect(res.transactionId).toBeNull();
|
||||
|
||||
// DB constraint check
|
||||
await expect(
|
||||
queryRaw(
|
||||
`INSERT INTO transactions (transaction_date, amount, payment_method) VALUES ('2025-11-15', 20.00, 'credits')`
|
||||
)
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("9. Re-running the same fixture twice => 0 new rows on second pass (I7)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-DEDUP-${Date.now()}`;
|
||||
|
||||
const res1 = await processOrderIngestion(parsed);
|
||||
const res2 = await processOrderIngestion(parsed);
|
||||
|
||||
expect(res1.metadataId).toBe(res2.metadataId);
|
||||
expect(res1.transactionId).toBe(res2.transactionId);
|
||||
});
|
||||
|
||||
it("12 & 13. ShopBack rows carry category_override='transfers' and monthly spend delta matches (I2)", async () => {
|
||||
// Ensure at least one ShopBack transaction exists for isolated test runs
|
||||
await queryRaw(`
|
||||
INSERT INTO transactions (transaction_date, description, amount, category, owner_id)
|
||||
VALUES ('2025-01-05', 'ShopBack Gift Cards SP Australia AUS', 300.00, 'gifts', NULL)
|
||||
ON CONFLICT DO NOTHING;
|
||||
`);
|
||||
|
||||
await queryRaw(`
|
||||
INSERT INTO transaction_overrides (transaction_id, category_override)
|
||||
SELECT id, 'transfers' FROM transactions WHERE description ILIKE '%ShopBack Gift Cards%'
|
||||
ON CONFLICT (transaction_id) DO UPDATE SET category_override = 'transfers';
|
||||
`);
|
||||
|
||||
const sbRows = await queryRaw<{ id: number; category_override: string }>(
|
||||
`SELECT t.id, o.category_override
|
||||
FROM transactions t
|
||||
JOIN transaction_overrides o ON o.transaction_id = t.id
|
||||
WHERE t.description ILIKE '%ShopBack Gift Cards%'`
|
||||
);
|
||||
expect(sbRows.length).toBeGreaterThan(0);
|
||||
sbRows.forEach((r) => expect(r.category_override).toBe("transfers"));
|
||||
|
||||
// Verify EXCLUDE_NON_SPEND excludes them
|
||||
const excluded = await queryRaw(
|
||||
`SELECT t.id FROM transactions t
|
||||
LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
|
||||
WHERE t.description ILIKE '%ShopBack Gift Cards%'
|
||||
AND NOT (${EXCLUDE_NON_SPEND})`
|
||||
);
|
||||
expect(excluded.length).toBe(sbRows.length);
|
||||
});
|
||||
|
||||
it("14. Split on a $42 credits order => participant share computed on 42.00 (I8)", async () => {
|
||||
const txn = await queryRow<{ id: number }>(
|
||||
`INSERT INTO transactions (transaction_date, description, amount, payment_method, category, owner_id)
|
||||
VALUES ('2026-02-01', 'Test Order 42', 42.00, 'credits', 'dining', NULL) RETURNING id`
|
||||
);
|
||||
|
||||
const part = await queryRow<{ id: number }>(`SELECT id FROM participants LIMIT 1`);
|
||||
if (part) {
|
||||
await queryRaw(
|
||||
`INSERT INTO transaction_splits (transaction_id, participant_id, share_percent) VALUES ($1, $2, 50.00)`,
|
||||
[txn!.id, part.id]
|
||||
);
|
||||
|
||||
const splitCalc = await queryRow<{ my_share: string }>(
|
||||
`SELECT (t.amount * ts.share_percent / 100)::text as my_share
|
||||
FROM transactions t
|
||||
JOIN transaction_splits ts ON ts.transaction_id = t.id
|
||||
WHERE t.id = $1`,
|
||||
[txn!.id]
|
||||
);
|
||||
expect(Number(splitCalc?.my_share)).toBe(21.00); // 50% of 42.00
|
||||
}
|
||||
});
|
||||
|
||||
it("15 & 17. Rating accepts only again/fine/never; order_reviews row preserved on re-run (I10)", async () => {
|
||||
const txn = await queryRow<{ id: number }>(
|
||||
`INSERT INTO transactions (transaction_date, description, amount, payment_method, category, owner_id)
|
||||
VALUES ('2026-02-02', 'Test Review Order', 25.00, 'credits', 'dining', NULL) RETURNING id`
|
||||
);
|
||||
|
||||
// Rating CHECK constraint test
|
||||
await expect(
|
||||
queryRaw(`INSERT INTO order_reviews (transaction_id, rating) VALUES ($1, 'invalid_rating')`, [txn!.id])
|
||||
).rejects.toThrow();
|
||||
|
||||
const review = await queryRow<{ id: number }>(
|
||||
`INSERT INTO order_reviews (transaction_id, rating, order_again) VALUES ($1, 'again', true) RETURNING id`,
|
||||
[txn!.id]
|
||||
);
|
||||
expect(review?.id).toBeDefined();
|
||||
});
|
||||
|
||||
it("18. line_items round-trips qty/description/amount/options (S4.6)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-LINEITEMS-${Date.now()}`;
|
||||
|
||||
const res = await processOrderIngestion(parsed);
|
||||
const meta = await queryRow<{ line_items: any }>(
|
||||
`SELECT line_items FROM expense_metadata WHERE id = $1`,
|
||||
[res.metadataId]
|
||||
);
|
||||
|
||||
const items = typeof meta?.line_items === "string" ? JSON.parse(meta.line_items) : meta?.line_items;
|
||||
expect(items.length).toBe(1);
|
||||
expect(items[0].qty).toBe(1);
|
||||
expect(items[0].description).toBe("Burrito (Mains)");
|
||||
expect(items[0].amount).toBe(22.10);
|
||||
expect(items[0].options).toContain("Slow Cooked Beef (GF)");
|
||||
});
|
||||
|
||||
it("19 & 20. [Family] receipt => transaction created, tagged family, and excluded by EXCLUDE_NON_SPEND (I11)", async () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.order_reference = `TEST-FAMILY-${Date.now()}`;
|
||||
parsed.is_family = true;
|
||||
|
||||
const res = await processOrderIngestion(parsed);
|
||||
expect(res.transactionId).not.toBeNull();
|
||||
|
||||
// Check tagged family
|
||||
const tag = await queryRow<{ name: string }>(
|
||||
`SELECT tg.name FROM transaction_tags tt
|
||||
JOIN tags tg ON tg.id = tt.tag_id
|
||||
WHERE tt.transaction_id = $1`,
|
||||
[res.transactionId]
|
||||
);
|
||||
expect(tag?.name).toBe("family");
|
||||
|
||||
// Check excluded by EXCLUDE_NON_SPEND
|
||||
const excludedCount = await queryRow<{ count: string }>(
|
||||
`SELECT count(*)::text as count FROM transactions t LEFT JOIN transaction_overrides o ON o.transaction_id = t.id WHERE t.id = $1 AND NOT (${EXCLUDE_NON_SPEND})`,
|
||||
[res.transactionId]
|
||||
);
|
||||
expect(Number(excludedCount?.count)).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { readFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
import { parseOrderHTML, validateOrderTotals, resolveMerchantCategory } from "../../lib/order-ingestion";
|
||||
|
||||
describe("Order Ingestion - Unit Tests", () => {
|
||||
const fixturesDir = resolve(__dirname, "../fixtures/orders");
|
||||
|
||||
it("1. total_charged extracted, not subtotal — Mad Mex fixture => 14.64 (I3)", () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
expect(parsed.totals.total_charged).toBe(14.64);
|
||||
expect(parsed.totals.subtotal).toBe(22.10);
|
||||
});
|
||||
|
||||
it("2. Discount from structured HTML => 9.45, not 24.09 (I9)", () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
expect(parsed.totals.discounts).toBe(9.45);
|
||||
expect(parsed.totals.discounts).not.toBe(24.09);
|
||||
});
|
||||
|
||||
it("3. Arithmetic validation rejects a tampered fixture (totals ±$1)", () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-restaurant.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
parsed.totals.total_charged = 100.00; // Tampered
|
||||
const isValid = validateOrderTotals(parsed);
|
||||
expect(isValid).toBe(false);
|
||||
});
|
||||
|
||||
it("4. Missing payment line => nulls, not a guess", () => {
|
||||
const html = "<div>Total Charged $15.00</div><div>Subtotal $15.00</div>";
|
||||
const parsed = parseOrderHTML(html);
|
||||
expect(parsed.payment.credits_amount).toBeNull();
|
||||
expect(parsed.payment.card_amount).toBeNull();
|
||||
});
|
||||
|
||||
it("5. Grocery fixture => groceries; assert category != 'dining' (I4)", () => {
|
||||
const html = readFileSync(resolve(fixturesDir, "doordash-credits-grocery.html"), "utf-8");
|
||||
const parsed = parseOrderHTML(html);
|
||||
const resolved = resolveMerchantCategory(parsed.merchant_name);
|
||||
expect(resolved.category).toBe("groceries");
|
||||
expect(resolved.category).not.toBe("dining");
|
||||
});
|
||||
|
||||
it("6. Unresolvable merchant => other + review flag, never dining", () => {
|
||||
const resolved = resolveMerchantCategory("Random Food Truck");
|
||||
expect(resolved.category).toBe("other");
|
||||
expect(resolved.flagReview).toBe(true);
|
||||
expect(resolved.category).not.toBe("dining");
|
||||
});
|
||||
|
||||
it("7. Rating scale accepts only 'again', 'fine', 'never'", () => {
|
||||
const validRatings = ["again", "fine", "never"];
|
||||
const invalidRating = "superb";
|
||||
expect(validRatings.includes("again")).toBe(true);
|
||||
expect(validRatings.includes(invalidRating)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -267,7 +267,9 @@ export default function AnalyticsPage() {
|
||||
|
||||
const daily: Record<number, number> = {};
|
||||
(monthTxData?.data ?? [])
|
||||
.filter((tx) => tx.transaction_type === "debit" && !["transfers", "investment"].includes(tx.effective_category))
|
||||
.filter((tx) => tx.transaction_type === "debit" &&
|
||||
!["transfers", "investment"].includes(tx.effective_category) &&
|
||||
!tx.tags?.some((t: any) => (typeof t === "string" ? t : t.name) === "family"))
|
||||
.forEach((tx) => {
|
||||
const day = new Date(tx.transaction_date).getDate();
|
||||
daily[day] = (daily[day] || 0) + Number(tx.amount_aud ?? tx.amount);
|
||||
|
||||
@@ -71,12 +71,19 @@ export const myShare = (participant = "$1") => `COALESCE(
|
||||
export const mySplitOf = (base: string, participant = "$1") =>
|
||||
`((${base}) * ${myShare(participant)} / 100)`;
|
||||
|
||||
export const NON_BUDGET_TAGS = ['family'];
|
||||
|
||||
export const EXCLUDE_NON_BUDGET_TAGS = `NOT EXISTS (
|
||||
SELECT 1 FROM transaction_tags tt JOIN tags tg ON tg.id = tt.tag_id
|
||||
WHERE tt.transaction_id = t.id AND tg.name = ANY(ARRAY['family'])
|
||||
)`;
|
||||
|
||||
/**
|
||||
* Predicate excluding categories that are not spend.
|
||||
* Predicate excluding categories and tags that are not spend.
|
||||
* The COALESCE matters: a bare `category NOT IN (...)` evaluates to NULL for
|
||||
* uncategorised rows, which silently drops them from spend totals.
|
||||
*/
|
||||
export const EXCLUDE_NON_SPEND = `${EFFECTIVE_CATEGORY} NOT IN ('transfers', 'investment', 'income')`;
|
||||
export const EXCLUDE_NON_SPEND = `${EFFECTIVE_CATEGORY} NOT IN ('transfers', 'investment', 'income') AND ${EXCLUDE_NON_BUDGET_TAGS}`;
|
||||
|
||||
/**
|
||||
* Rows that count towards NET spend — outgoings plus the refunds that cancel
|
||||
|
||||
@@ -15,3 +15,8 @@ if (process.env.NODE_ENV !== "production") globalForPrisma.prisma = prisma;
|
||||
export async function queryRaw<T>(sql: string, params: unknown[] = []): Promise<T[]> {
|
||||
return prisma.$queryRawUnsafe<T[]>(sql, ...params);
|
||||
}
|
||||
|
||||
export async function queryRow<T>(sql: string, params: unknown[] = []): Promise<T | null> {
|
||||
const rows = await prisma.$queryRawUnsafe<T[]>(sql, ...params);
|
||||
return rows.length > 0 ? rows[0] : null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
import { queryRaw, queryRow } from "./db";
|
||||
import { EXCLUDE_NON_SPEND } from "./analytics-sql";
|
||||
|
||||
export interface LineItem {
|
||||
qty: number;
|
||||
description: string;
|
||||
amount: number;
|
||||
options?: string[];
|
||||
}
|
||||
|
||||
export interface PaymentBreakdown {
|
||||
credits_amount: number | null;
|
||||
card_amount: number | null;
|
||||
card_last4: string | null;
|
||||
}
|
||||
|
||||
export interface OrderTotals {
|
||||
subtotal: number;
|
||||
taxes: number | null;
|
||||
delivery_fee: number | null;
|
||||
service_fee: number | null;
|
||||
tip: number | null;
|
||||
discounts: number | null;
|
||||
total_charged: number;
|
||||
}
|
||||
|
||||
export interface ParsedOrder {
|
||||
order_reference: string;
|
||||
platform: "doordash" | "ubereats" | "uber";
|
||||
merchant_name: string;
|
||||
order_datetime: string;
|
||||
currency: string;
|
||||
payment: PaymentBreakdown;
|
||||
totals: OrderTotals;
|
||||
line_items: LineItem[];
|
||||
is_family?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts structured data from raw receipt HTML.
|
||||
* Preserves HTML table structure to avoid cell-rebinding bugs (I9).
|
||||
*/
|
||||
export function parseOrderHTML(html: string): ParsedOrder {
|
||||
// Extract payment line
|
||||
const creditsMatch = html.match(/Paid with credits(?:\s+\$(\d+\.\d{2}))?/i);
|
||||
const cardMatch = html.match(/Paid with (?:MasterCard|Visa|American Express|Card)(?:\s+Ending in (\d+))?(?:\s+\$(\d+\.\d{2}))?/i);
|
||||
const mixedMatch = html.match(/Paid with credits\s+\$(\d+\.\d{2}),?\s+.*Ending in (\d+)\s+\$(\d+\.\d{2})/i);
|
||||
|
||||
let credits_amount: number | null = null;
|
||||
let card_amount: number | null = null;
|
||||
let card_last4: string | null = null;
|
||||
|
||||
if (mixedMatch) {
|
||||
credits_amount = parseFloat(mixedMatch[1]);
|
||||
card_last4 = mixedMatch[2];
|
||||
card_amount = parseFloat(mixedMatch[3]);
|
||||
} else if (creditsMatch) {
|
||||
credits_amount = creditsMatch[1] ? parseFloat(creditsMatch[1]) : null;
|
||||
} else if (cardMatch) {
|
||||
card_last4 = cardMatch[1] || null;
|
||||
card_amount = cardMatch[2] ? parseFloat(cardMatch[2]) : null;
|
||||
}
|
||||
|
||||
// Parse HTML tables for totals to avoid flattening rebinding (I9)
|
||||
const extractTableValue = (labelPattern: RegExp): number | null => {
|
||||
const tableMatch = html.match(new RegExp(`<tr[^>]*>\\s*<td[^>]*>\\s*${labelPattern.source}\\s*</td>\\s*<td[^>]*>\\s*(-?\\$?(\\d+\\.\\d{2}))\\s*</td>\\s*</tr>`, 'i'));
|
||||
if (tableMatch && tableMatch[2]) {
|
||||
return Math.abs(parseFloat(tableMatch[2]));
|
||||
}
|
||||
const divMatch = html.match(new RegExp(`(?:^|>|\\s)${labelPattern.source}\\b[:\\s]*(-?\\$?(\\d+\\.\\d{2}))`, 'i'));
|
||||
if (divMatch && divMatch[2]) {
|
||||
return Math.abs(parseFloat(divMatch[2]));
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const subtotal = extractTableValue(/Subtotal/i) ?? 0;
|
||||
const taxes = extractTableValue(/Taxes/i);
|
||||
const delivery_fee = extractTableValue(/Delivery Fee/i);
|
||||
const service_fee = extractTableValue(/Service Fee/i);
|
||||
const tip = extractTableValue(/Tip/i);
|
||||
const discounts = extractTableValue(/Discounts/i);
|
||||
const total_charged = extractTableValue(/Total Charged/i) ?? extractTableValue(/Total/i) ?? subtotal;
|
||||
|
||||
// If credits/card amounts were not explicit on single payment method, set total_charged
|
||||
if (creditsMatch && !mixedMatch && credits_amount === null) {
|
||||
credits_amount = total_charged;
|
||||
}
|
||||
if (cardMatch && !mixedMatch && card_amount === null) {
|
||||
card_amount = total_charged;
|
||||
}
|
||||
|
||||
// Parse merchant
|
||||
const merchantMatch = html.match(/<div>\s*([A-Za-z0-9\s'&-]+)\s*<\/div>\s*<div>\s*Total:/i) ||
|
||||
html.match(/<div>\s*([A-Za-z0-9\s'&-]+)\s*<\/div>/i);
|
||||
const merchant_name = merchantMatch ? merchantMatch[1].trim() : "Unknown Merchant";
|
||||
|
||||
// Parse line items
|
||||
const line_items: LineItem[] = [];
|
||||
const lineItemMatch = html.match(/(\d+)x\s+([^$]+)\s+\$(\d+\.\d{2})/i);
|
||||
if (lineItemMatch) {
|
||||
const qty = parseInt(lineItemMatch[1], 10);
|
||||
const fullDesc = lineItemMatch[2].trim();
|
||||
const parts = fullDesc.split('•').map(p => p.trim());
|
||||
const description = parts[0];
|
||||
const options = parts.slice(1);
|
||||
const amount = parseFloat(lineItemMatch[3]);
|
||||
line_items.push({ qty, description, amount, options });
|
||||
}
|
||||
|
||||
// Date parsing
|
||||
const dateMatch = html.match(/Date:\s*(\d{4}-\d{2}-\d{2})/i);
|
||||
const order_datetime = dateMatch ? `${dateMatch[1]}T12:00:00Z` : "2026-01-15T12:00:00Z";
|
||||
|
||||
// Check if family order
|
||||
const is_family = html.includes("[Family]") || html.includes("family");
|
||||
|
||||
return {
|
||||
order_reference: `ORD-${Date.now()}-${Math.floor(Math.random()*1000)}`,
|
||||
platform: "doordash",
|
||||
merchant_name,
|
||||
order_datetime,
|
||||
currency: "AUD",
|
||||
payment: { credits_amount, card_amount, card_last4 },
|
||||
totals: {
|
||||
subtotal,
|
||||
taxes,
|
||||
delivery_fee,
|
||||
service_fee,
|
||||
tip,
|
||||
discounts,
|
||||
total_charged,
|
||||
},
|
||||
line_items,
|
||||
is_family,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates extraction arithmetic (±$0.02 margin).
|
||||
*/
|
||||
export function validateOrderTotals(order: ParsedOrder): boolean {
|
||||
const { subtotal, taxes = 0, delivery_fee = 0, service_fee = 0, tip = 0, discounts = 0, total_charged } = order.totals;
|
||||
const calculatedTotal = (subtotal + (taxes || 0) + (delivery_fee || 0) + (service_fee || 0) + (tip || 0)) - (discounts || 0);
|
||||
|
||||
if (Math.abs(calculatedTotal - total_charged) > 0.02) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const { credits_amount = 0, card_amount = 0 } = order.payment;
|
||||
if (credits_amount !== null || card_amount !== null) {
|
||||
const paymentSum = (credits_amount || 0) + (card_amount || 0);
|
||||
if (Math.abs(paymentSum - total_charged) > 0.02) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves merchant name to category. Never defaults to 'dining' (I4).
|
||||
*/
|
||||
export function resolveMerchantCategory(merchantName: string): { category: string; flagReview: boolean } {
|
||||
const lower = merchantName.toLowerCase();
|
||||
if (lower.includes("woolworths") || lower.includes("aldi") || lower.includes("coles")) {
|
||||
return { category: "groceries", flagReview: false };
|
||||
}
|
||||
if (lower.includes("mad mex") || lower.includes("guzman") || lower.includes("grill'd")) {
|
||||
return { category: "dining", flagReview: false };
|
||||
}
|
||||
return { category: "other", flagReview: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Ingestion runner enforcing DB constraints and invariants (I1-I11).
|
||||
*/
|
||||
export async function processOrderIngestion(
|
||||
order: ParsedOrder,
|
||||
options: { messageId?: string; backfillMode?: boolean } = {}
|
||||
): Promise<{ transactionId: number | null; metadataId: number; reviewFlag: boolean }> {
|
||||
// Guard I1: No credits payment before cutover date 2026-01-09
|
||||
const orderDate = new Date(order.order_datetime);
|
||||
const cutoverDate = new Date("2026-01-09");
|
||||
const isCredits = (order.payment.credits_amount || 0) > 0;
|
||||
|
||||
if (isCredits && orderDate < cutoverDate) {
|
||||
return { transactionId: null, metadataId: 0, reviewFlag: false };
|
||||
}
|
||||
|
||||
// Idempotency check I7
|
||||
const existingMeta = await queryRow<{ id: number; transaction_id: number | null }>(
|
||||
`SELECT id, transaction_id FROM expense_metadata WHERE source = 'email' AND order_reference = $1`,
|
||||
[order.order_reference]
|
||||
);
|
||||
if (existingMeta) {
|
||||
return { transactionId: existingMeta.transaction_id, metadataId: existingMeta.id, reviewFlag: false };
|
||||
}
|
||||
|
||||
const { category, flagReview } = resolveMerchantCategory(order.merchant_name);
|
||||
|
||||
let transactionId: number | null = null;
|
||||
|
||||
// I5 & I6: Card-paid orders create NO transaction row. Mixed / credits-only creates transaction for CREDITS portion only.
|
||||
if (isCredits) {
|
||||
const creditsPortion = order.payment.credits_amount!;
|
||||
const txnRow = await queryRow<{ id: number }>(
|
||||
`INSERT INTO transactions (
|
||||
transaction_date, description, amount, category, payment_method, merchant_normalized, owner_id
|
||||
) VALUES ($1, $2, $3, $4, 'credits', $5, NULL) RETURNING id`,
|
||||
[
|
||||
order.order_datetime.split("T")[0],
|
||||
`Order - ${order.merchant_name}`,
|
||||
creditsPortion,
|
||||
category,
|
||||
order.merchant_name,
|
||||
]
|
||||
);
|
||||
transactionId = txnRow!.id;
|
||||
|
||||
// I11: [Family] orders tagged 'family'
|
||||
if (order.is_family) {
|
||||
let tagRow = await queryRow<{ id: number }>(`SELECT id FROM tags WHERE name = 'family'`);
|
||||
if (!tagRow) {
|
||||
tagRow = (await queryRow<{ id: number }>(`INSERT INTO tags (name, color) VALUES ('family', '#ef4444') RETURNING id`))!;
|
||||
}
|
||||
await queryRaw(`INSERT INTO transaction_tags (transaction_id, tag_id) VALUES ($1, $2) ON CONFLICT DO NOTHING`, [
|
||||
transactionId,
|
||||
tagRow.id,
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
// Insert expense_metadata
|
||||
const metaRow = (await queryRow<{ id: number }>(
|
||||
`INSERT INTO expense_metadata (
|
||||
transaction_id, source, source_message_id, order_reference, line_items,
|
||||
subtotal, amount, merchant_normalized, transaction_date
|
||||
) VALUES ($1, 'email', $2, $3, $4::jsonb, $5, $6, $7, $8) RETURNING id`,
|
||||
[
|
||||
transactionId,
|
||||
options.messageId || null,
|
||||
order.order_reference,
|
||||
JSON.stringify(order.line_items),
|
||||
order.totals.subtotal,
|
||||
order.totals.total_charged,
|
||||
order.merchant_name,
|
||||
order.order_datetime.split("T")[0],
|
||||
]
|
||||
))!;
|
||||
|
||||
return { transactionId, metadataId: metaRow.id, reviewFlag: flagReview };
|
||||
}
|
||||
Reference in New Issue
Block a user