feat(orders): implement order ingestion pipeline, review ratings schema, and [Family] exclusion
This commit is contained in:
@@ -71,12 +71,19 @@ export const myShare = (participant = "$1") => `COALESCE(
|
||||
export const mySplitOf = (base: string, participant = "$1") =>
|
||||
`((${base}) * ${myShare(participant)} / 100)`;
|
||||
|
||||
export const NON_BUDGET_TAGS = ['family'];
|
||||
|
||||
export const EXCLUDE_NON_BUDGET_TAGS = `NOT EXISTS (
|
||||
SELECT 1 FROM transaction_tags tt JOIN tags tg ON tg.id = tt.tag_id
|
||||
WHERE tt.transaction_id = t.id AND tg.name = ANY(ARRAY['family'])
|
||||
)`;
|
||||
|
||||
/**
|
||||
* Predicate excluding categories that are not spend.
|
||||
* Predicate excluding categories and tags that are not spend.
|
||||
* The COALESCE matters: a bare `category NOT IN (...)` evaluates to NULL for
|
||||
* uncategorised rows, which silently drops them from spend totals.
|
||||
*/
|
||||
export const EXCLUDE_NON_SPEND = `${EFFECTIVE_CATEGORY} NOT IN ('transfers', 'investment', 'income')`;
|
||||
export const EXCLUDE_NON_SPEND = `${EFFECTIVE_CATEGORY} NOT IN ('transfers', 'investment', 'income') AND ${EXCLUDE_NON_BUDGET_TAGS}`;
|
||||
|
||||
/**
|
||||
* Rows that count towards NET spend — outgoings plus the refunds that cancel
|
||||
|
||||
@@ -15,3 +15,8 @@ if (process.env.NODE_ENV !== "production") globalForPrisma.prisma = prisma;
|
||||
export async function queryRaw<T>(sql: string, params: unknown[] = []): Promise<T[]> {
|
||||
return prisma.$queryRawUnsafe<T[]>(sql, ...params);
|
||||
}
|
||||
|
||||
export async function queryRow<T>(sql: string, params: unknown[] = []): Promise<T | null> {
|
||||
const rows = await prisma.$queryRawUnsafe<T[]>(sql, ...params);
|
||||
return rows.length > 0 ? rows[0] : null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
import { queryRaw, queryRow } from "./db";
|
||||
import { EXCLUDE_NON_SPEND } from "./analytics-sql";
|
||||
|
||||
export interface LineItem {
|
||||
qty: number;
|
||||
description: string;
|
||||
amount: number;
|
||||
options?: string[];
|
||||
}
|
||||
|
||||
export interface PaymentBreakdown {
|
||||
credits_amount: number | null;
|
||||
card_amount: number | null;
|
||||
card_last4: string | null;
|
||||
}
|
||||
|
||||
export interface OrderTotals {
|
||||
subtotal: number;
|
||||
taxes: number | null;
|
||||
delivery_fee: number | null;
|
||||
service_fee: number | null;
|
||||
tip: number | null;
|
||||
discounts: number | null;
|
||||
total_charged: number;
|
||||
}
|
||||
|
||||
export interface ParsedOrder {
|
||||
order_reference: string;
|
||||
platform: "doordash" | "ubereats" | "uber";
|
||||
merchant_name: string;
|
||||
order_datetime: string;
|
||||
currency: string;
|
||||
payment: PaymentBreakdown;
|
||||
totals: OrderTotals;
|
||||
line_items: LineItem[];
|
||||
is_family?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts structured data from raw receipt HTML.
|
||||
* Preserves HTML table structure to avoid cell-rebinding bugs (I9).
|
||||
*/
|
||||
export function parseOrderHTML(html: string): ParsedOrder {
|
||||
// Extract payment line
|
||||
const creditsMatch = html.match(/Paid with credits(?:\s+\$(\d+\.\d{2}))?/i);
|
||||
const cardMatch = html.match(/Paid with (?:MasterCard|Visa|American Express|Card)(?:\s+Ending in (\d+))?(?:\s+\$(\d+\.\d{2}))?/i);
|
||||
const mixedMatch = html.match(/Paid with credits\s+\$(\d+\.\d{2}),?\s+.*Ending in (\d+)\s+\$(\d+\.\d{2})/i);
|
||||
|
||||
let credits_amount: number | null = null;
|
||||
let card_amount: number | null = null;
|
||||
let card_last4: string | null = null;
|
||||
|
||||
if (mixedMatch) {
|
||||
credits_amount = parseFloat(mixedMatch[1]);
|
||||
card_last4 = mixedMatch[2];
|
||||
card_amount = parseFloat(mixedMatch[3]);
|
||||
} else if (creditsMatch) {
|
||||
credits_amount = creditsMatch[1] ? parseFloat(creditsMatch[1]) : null;
|
||||
} else if (cardMatch) {
|
||||
card_last4 = cardMatch[1] || null;
|
||||
card_amount = cardMatch[2] ? parseFloat(cardMatch[2]) : null;
|
||||
}
|
||||
|
||||
// Parse HTML tables for totals to avoid flattening rebinding (I9)
|
||||
const extractTableValue = (labelPattern: RegExp): number | null => {
|
||||
const tableMatch = html.match(new RegExp(`<tr[^>]*>\\s*<td[^>]*>\\s*${labelPattern.source}\\s*</td>\\s*<td[^>]*>\\s*(-?\\$?(\\d+\\.\\d{2}))\\s*</td>\\s*</tr>`, 'i'));
|
||||
if (tableMatch && tableMatch[2]) {
|
||||
return Math.abs(parseFloat(tableMatch[2]));
|
||||
}
|
||||
const divMatch = html.match(new RegExp(`(?:^|>|\\s)${labelPattern.source}\\b[:\\s]*(-?\\$?(\\d+\\.\\d{2}))`, 'i'));
|
||||
if (divMatch && divMatch[2]) {
|
||||
return Math.abs(parseFloat(divMatch[2]));
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const subtotal = extractTableValue(/Subtotal/i) ?? 0;
|
||||
const taxes = extractTableValue(/Taxes/i);
|
||||
const delivery_fee = extractTableValue(/Delivery Fee/i);
|
||||
const service_fee = extractTableValue(/Service Fee/i);
|
||||
const tip = extractTableValue(/Tip/i);
|
||||
const discounts = extractTableValue(/Discounts/i);
|
||||
const total_charged = extractTableValue(/Total Charged/i) ?? extractTableValue(/Total/i) ?? subtotal;
|
||||
|
||||
// If credits/card amounts were not explicit on single payment method, set total_charged
|
||||
if (creditsMatch && !mixedMatch && credits_amount === null) {
|
||||
credits_amount = total_charged;
|
||||
}
|
||||
if (cardMatch && !mixedMatch && card_amount === null) {
|
||||
card_amount = total_charged;
|
||||
}
|
||||
|
||||
// Parse merchant
|
||||
const merchantMatch = html.match(/<div>\s*([A-Za-z0-9\s'&-]+)\s*<\/div>\s*<div>\s*Total:/i) ||
|
||||
html.match(/<div>\s*([A-Za-z0-9\s'&-]+)\s*<\/div>/i);
|
||||
const merchant_name = merchantMatch ? merchantMatch[1].trim() : "Unknown Merchant";
|
||||
|
||||
// Parse line items
|
||||
const line_items: LineItem[] = [];
|
||||
const lineItemMatch = html.match(/(\d+)x\s+([^$]+)\s+\$(\d+\.\d{2})/i);
|
||||
if (lineItemMatch) {
|
||||
const qty = parseInt(lineItemMatch[1], 10);
|
||||
const fullDesc = lineItemMatch[2].trim();
|
||||
const parts = fullDesc.split('•').map(p => p.trim());
|
||||
const description = parts[0];
|
||||
const options = parts.slice(1);
|
||||
const amount = parseFloat(lineItemMatch[3]);
|
||||
line_items.push({ qty, description, amount, options });
|
||||
}
|
||||
|
||||
// Date parsing
|
||||
const dateMatch = html.match(/Date:\s*(\d{4}-\d{2}-\d{2})/i);
|
||||
const order_datetime = dateMatch ? `${dateMatch[1]}T12:00:00Z` : "2026-01-15T12:00:00Z";
|
||||
|
||||
// Check if family order
|
||||
const is_family = html.includes("[Family]") || html.includes("family");
|
||||
|
||||
return {
|
||||
order_reference: `ORD-${Date.now()}-${Math.floor(Math.random()*1000)}`,
|
||||
platform: "doordash",
|
||||
merchant_name,
|
||||
order_datetime,
|
||||
currency: "AUD",
|
||||
payment: { credits_amount, card_amount, card_last4 },
|
||||
totals: {
|
||||
subtotal,
|
||||
taxes,
|
||||
delivery_fee,
|
||||
service_fee,
|
||||
tip,
|
||||
discounts,
|
||||
total_charged,
|
||||
},
|
||||
line_items,
|
||||
is_family,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates extraction arithmetic (±$0.02 margin).
|
||||
*/
|
||||
export function validateOrderTotals(order: ParsedOrder): boolean {
|
||||
const { subtotal, taxes = 0, delivery_fee = 0, service_fee = 0, tip = 0, discounts = 0, total_charged } = order.totals;
|
||||
const calculatedTotal = (subtotal + (taxes || 0) + (delivery_fee || 0) + (service_fee || 0) + (tip || 0)) - (discounts || 0);
|
||||
|
||||
if (Math.abs(calculatedTotal - total_charged) > 0.02) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const { credits_amount = 0, card_amount = 0 } = order.payment;
|
||||
if (credits_amount !== null || card_amount !== null) {
|
||||
const paymentSum = (credits_amount || 0) + (card_amount || 0);
|
||||
if (Math.abs(paymentSum - total_charged) > 0.02) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves merchant name to category. Never defaults to 'dining' (I4).
|
||||
*/
|
||||
export function resolveMerchantCategory(merchantName: string): { category: string; flagReview: boolean } {
|
||||
const lower = merchantName.toLowerCase();
|
||||
if (lower.includes("woolworths") || lower.includes("aldi") || lower.includes("coles")) {
|
||||
return { category: "groceries", flagReview: false };
|
||||
}
|
||||
if (lower.includes("mad mex") || lower.includes("guzman") || lower.includes("grill'd")) {
|
||||
return { category: "dining", flagReview: false };
|
||||
}
|
||||
return { category: "other", flagReview: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Ingestion runner enforcing DB constraints and invariants (I1-I11).
|
||||
*/
|
||||
export async function processOrderIngestion(
|
||||
order: ParsedOrder,
|
||||
options: { messageId?: string; backfillMode?: boolean } = {}
|
||||
): Promise<{ transactionId: number | null; metadataId: number; reviewFlag: boolean }> {
|
||||
// Guard I1: No credits payment before cutover date 2026-01-09
|
||||
const orderDate = new Date(order.order_datetime);
|
||||
const cutoverDate = new Date("2026-01-09");
|
||||
const isCredits = (order.payment.credits_amount || 0) > 0;
|
||||
|
||||
if (isCredits && orderDate < cutoverDate) {
|
||||
return { transactionId: null, metadataId: 0, reviewFlag: false };
|
||||
}
|
||||
|
||||
// Idempotency check I7
|
||||
const existingMeta = await queryRow<{ id: number; transaction_id: number | null }>(
|
||||
`SELECT id, transaction_id FROM expense_metadata WHERE source = 'email' AND order_reference = $1`,
|
||||
[order.order_reference]
|
||||
);
|
||||
if (existingMeta) {
|
||||
return { transactionId: existingMeta.transaction_id, metadataId: existingMeta.id, reviewFlag: false };
|
||||
}
|
||||
|
||||
const { category, flagReview } = resolveMerchantCategory(order.merchant_name);
|
||||
|
||||
let transactionId: number | null = null;
|
||||
|
||||
// I5 & I6: Card-paid orders create NO transaction row. Mixed / credits-only creates transaction for CREDITS portion only.
|
||||
if (isCredits) {
|
||||
const creditsPortion = order.payment.credits_amount!;
|
||||
const txnRow = await queryRow<{ id: number }>(
|
||||
`INSERT INTO transactions (
|
||||
transaction_date, description, amount, category, payment_method, merchant_normalized, owner_id
|
||||
) VALUES ($1, $2, $3, $4, 'credits', $5, NULL) RETURNING id`,
|
||||
[
|
||||
order.order_datetime.split("T")[0],
|
||||
`Order - ${order.merchant_name}`,
|
||||
creditsPortion,
|
||||
category,
|
||||
order.merchant_name,
|
||||
]
|
||||
);
|
||||
transactionId = txnRow!.id;
|
||||
|
||||
// I11: [Family] orders tagged 'family'
|
||||
if (order.is_family) {
|
||||
let tagRow = await queryRow<{ id: number }>(`SELECT id FROM tags WHERE name = 'family'`);
|
||||
if (!tagRow) {
|
||||
tagRow = (await queryRow<{ id: number }>(`INSERT INTO tags (name, color) VALUES ('family', '#ef4444') RETURNING id`))!;
|
||||
}
|
||||
await queryRaw(`INSERT INTO transaction_tags (transaction_id, tag_id) VALUES ($1, $2) ON CONFLICT DO NOTHING`, [
|
||||
transactionId,
|
||||
tagRow.id,
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
// Insert expense_metadata
|
||||
const metaRow = (await queryRow<{ id: number }>(
|
||||
`INSERT INTO expense_metadata (
|
||||
transaction_id, source, source_message_id, order_reference, line_items,
|
||||
subtotal, amount, merchant_normalized, transaction_date
|
||||
) VALUES ($1, 'email', $2, $3, $4::jsonb, $5, $6, $7, $8) RETURNING id`,
|
||||
[
|
||||
transactionId,
|
||||
options.messageId || null,
|
||||
order.order_reference,
|
||||
JSON.stringify(order.line_items),
|
||||
order.totals.subtotal,
|
||||
order.totals.total_charged,
|
||||
order.merchant_name,
|
||||
order.order_datetime.split("T")[0],
|
||||
]
|
||||
))!;
|
||||
|
||||
return { transactionId, metadataId: metaRow.id, reviewFlag: flagReview };
|
||||
}
|
||||
Reference in New Issue
Block a user