finance-app/src/lib/order-ingestion.ts

import { queryRaw, queryRow } from "./db";
import type { ParsedOrder } from "./order-parse";

export * from "./order-parse";

export const CUTOVER_DATE = "2026-01-09";

/**
 * Owner for ingested orders. Analytics scope on COALESCE(t.owner_id,
 * s.owner_id); an ingested order has no statement, so leaving owner_id NULL
 * hides it from every view in the app while it sits in the table.
 */
export const DEFAULT_OWNER_ID = 1;

/**
 * Transaction description.
 *
 * The merchant alone ("Order - Burger Corner") does not say where to go and
 * look for the detail, and there are restaurants on both platforms. The
 * platform is the one thing the parser always knows and used to discard.
 */
export function orderDescription(_platform: ParsedOrder["platform"], merchant: string): string {
  // The platform is deliberately NOT in the headline. What identifies the
  // transaction is the restaurant; which courier delivered it is provenance,
  // and it already has a home — the expandable Order details panel renders
  // `expense_metadata.platform` next to its heading (user, 2026-07-28).
  //
  // Keeping it here also fragmented the merchant: the same restaurant reads
  // "Mad Mex (DoorDash)" and "Mad Mex (Uber Eats)" depending on who carried
  // the bag, which is not a distinction anyone rating the food cares about.
  return `Order - ${merchant}`;
}

export interface IngestResult {
  transactionId: number | null;
  metadataId: number | null;
  flags: string[];
  skipped?: string;
}

/**
 * Resolves how much of a card-settled order actually hit the card.
 *
 * DoorDash writes "MasterCard Ending in 8032 and/or credits" without ever
 * stating the split. The split is not in the mail — but it IS in the ledger:
 * the card leg arrives on the statement for that card. Reconciling against it
 * beats guessing (user, 2026-07-26).
 *
 *   Subway  receipt $29.08, statement 8032 charge $13.06 -> $16.02 was credits
 *   lahori  receipt $50.85, statement 8032 charge $50.85 -> fully card
 *
 * Returns the card amount if a statement line can be matched, else null. A null
 * means "unknown", and the caller must not invent a credits figure from it.
 */
export async function reconcileCardLeg(
  order: ParsedOrder,
  windowDays = 4
): Promise<{ cardAmount: number | null; matchedTransactionId: number | null }> {
  const last4 = order.payment.card_last4;
  if (!last4) return { cardAmount: null, matchedTransactionId: null };

  const day = order.order_datetime.slice(0, 10);
  const row = await queryRow<{ id: number; amount: string }>(
    `SELECT t.id, t.amount::text
       FROM transactions t
       JOIN statements s ON s.id = t.statement_id
      WHERE replace(s.account_number, '-', '') LIKE $1
        AND t.transaction_date BETWEEN $2::date - $4::int AND $2::date + $4::int
        AND (t.description ILIKE '%doordash%' OR t.description ILIKE '%uber%')
        AND t.amount <= $3::numeric + 0.02
        -- A statement line settles exactly one order. Without this, two orders
        -- on the same card inside the window both match the same charge and
        -- each books its own credits remainder — double-counting spend. At
        -- 10-15 orders a month on one card that is not a corner case.
        AND NOT EXISTS (
          SELECT 1 FROM expense_metadata em
           WHERE em.matched_transaction_id = t.id
             AND ($5::text IS NULL OR em.order_reference IS DISTINCT FROM $5::text)
        )
      ORDER BY abs(t.amount - $3::numeric), abs(t.transaction_date - $2::date)
      LIMIT 1`,
    [`%${last4}`, day, order.totals.total_charged, windowDays, order.order_reference || null]
  );

  if (!row) return { cardAmount: null, matchedTransactionId: null };
  return { cardAmount: Number(row.amount), matchedTransactionId: row.id };
}

async function ensureTag(name: string): Promise<number> {
  const existing = await queryRow<{ id: number }>(`SELECT id FROM tags WHERE name = $1`, [name]);
  if (existing) return existing.id;
  const created = await queryRow<{ id: number }>(
    `INSERT INTO tags (name, color) VALUES ($1, '#ef4444')
     ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name RETURNING id`,
    [name]
  );
  return created!.id;
}

/**
 * Records one parsed order.
 *
 * Invariants enforced here:
 *  I1  no credits-funded order before the cutover — before it, splits lived in
 *      another system and re-importing double-counts.
 *  I5  a card-settled order creates NO transaction. The statement line is the
 *      transaction; creating another would double-count.
 *  I6  a credits-funded order creates a transaction for the credits portion at
 *      face value.
 *  I7  idempotent on (source, order_reference).
 *  I11 [Family] orders are imported and tagged, never silently dropped.
 */
export async function processOrderIngestion(
  order: ParsedOrder,
  options: {
    messageId?: string;
    backfillMode?: boolean;
    ownerId?: number;
    subject?: string;
    sender?: string;
  } = {}
): Promise<IngestResult> {
  const flags = [...order.flags];
  const day = order.order_datetime.slice(0, 10);

  // ---- I7: idempotency ----------------------------------------------------
  const existing = await queryRow<{ id: number; transaction_id: number | null }>(
    `SELECT id, transaction_id FROM expense_metadata
      WHERE source = 'email' AND order_reference = $1`,
    [order.order_reference]
  );
  if (existing) {
    return {
      transactionId: existing.transaction_id,
      metadataId: existing.id,
      flags,
      skipped: "already_ingested",
    };
  }

  // ---- resolve the credits portion ----------------------------------------
  let creditsAmount: number | null = null;

  if (order.payment.ambiguous) {
    const { cardAmount: reconciled } = await reconcileCardLeg(order);
    if (reconciled === null) {
      // No statement line yet. For a live order this is the NORMAL case, not an
      // error — card statements arrive monthly, so an order ingested today has
      // no card leg in the ledger for weeks (user, 2026-07-26).
      //
      // Deciding now would mean guessing. Instead the order is recorded as
      // provenance with no transaction, and left pending: reconcilePendingOrders()
      // resolves it once the statement lands. Backfill hits the same path and
      // resolves immediately, because those statements are already imported.
      flags.push("awaiting_card_statement");
    } else {
      const remainder = Number((order.totals.total_charged - reconciled).toFixed(2));
      if (remainder > 0.02) {
        creditsAmount = remainder;
        flags.push(`split_reconciled_card_${reconciled.toFixed(2)}`);
      }
    }
  } else {
    creditsAmount = order.payment.credits_amount;
  }

  // ---- I1: cutover (retired 2026-07-28) -----------------------------------
  //
  // This used to return early for any credits order dated before the cutover,
  // storing NOTHING — no transaction and no metadata, so the receipt was
  // discarded entirely.
  //
  // Its reason was splits, not spend: before 2026-01-09 shared expenses lived
  // in SplitMyExpenses, and re-importing them would double-charge against
  // carryover transaction 2348. That reason expired with 788219b, where
  // ACTIVE_OBLIGATION became `settled = false AND transaction_date >=
  // '2026-01-09'` — a pre-cutover split can no longer assert a debt, so a
  // pre-cutover order cannot move a balance however it is recorded.
  //
  // What it was left doing was hiding ordinary history: 275 orders, $9,799.96
  // of meals and rides across 2020-2025, invisible purely because they were
  // paid from a gift-card balance instead of a card. Ingestion still writes no
  // splits at all, so nothing here touches what anyone owes.
  //
  // The known imprecision is on the funding side and is accepted rather than
  // guessed at — see migration 0026 for why the ShopBack loads are left as
  // they are.
  if (creditsAmount !== null && day < CUTOVER_DATE) {
    flags.push("pre_cutover_credits_order");
  }

  // ---- I6 / I5 ------------------------------------------------------------
  let transactionId: number | null = null;
  if (creditsAmount !== null && creditsAmount > 0) {
    const isAud = order.currency === "AUD";
    if (!isAud) flags.push(`foreign_currency_${order.currency}`);

    const txn = await queryRow<{ id: number }>(
      `INSERT INTO transactions (
         transaction_date, description, amount, amount_aud, category, payment_method,
         merchant_name, merchant_normalized, transaction_type,
         foreign_currency_amount, foreign_currency_code, owner_id
       ) VALUES ($1,$2,$3,$4,$5,'credits',$6,$6,'debit',$7,$8,$9)
       RETURNING id`,
      [
        day,
        orderDescription(order.platform, order.merchant_name),
        creditsAmount,
        // No FX rate is available at ingest, so amount_aud is left NULL for
        // foreign orders rather than asserting a conversion we cannot make.
        isAud ? creditsAmount : null,
        resolveCategory(order),
        order.merchant_name,
        isAud ? null : creditsAmount,
        isAud ? null : order.currency,
        // Owner scoping is COALESCE(t.owner_id, s.owner_id). These rows carry
        // no statement, so a NULL owner_id makes them invisible in every view
        // in the app — present in the table, absent from the UI. The backfill
        // inserted 85 rows nobody could see.
        options.ownerId ?? DEFAULT_OWNER_ID,
      ]
    );
    transactionId = txn!.id;

    // I11: tag, don't drop. The tag is what removes it from budgets.
    if (order.is_family) {
      const tagId = await ensureTag("family");
      await queryRaw(
        `INSERT INTO transaction_tags (transaction_id, tag_id) VALUES ($1,$2)
         ON CONFLICT DO NOTHING`,
        [transactionId, tagId]
      );
    }
  }

  // ---- provenance ---------------------------------------------------------
  const pending = flags.includes("awaiting_card_statement");
  const meta = await queryRow<{ id: number }>(
    `INSERT INTO expense_metadata (
       transaction_id, source, source_message_id, order_reference, line_items,
       subtotal, amount, merchant_normalized, transaction_date,
       card_last4, currency, flags, reconciled_at,
       platform, source_email_subject, source_email_from, route
     ) VALUES ($1,'email',$2,$3,$4::jsonb,$5,$6,$7,$8,$9,$10,$11::jsonb,$12,$13,$14,$15,$16::jsonb)
     RETURNING id`,
    [
      transactionId,
      options.messageId || null,
      order.order_reference,
      JSON.stringify(order.line_items),
      order.totals.subtotal,
      order.totals.total_charged,
      order.merchant_name,
      day,
      order.payment.card_last4,
      order.currency,
      JSON.stringify(flags),
      pending ? null : new Date().toISOString(),
      order.platform,
      options.subject ?? null,
      options.sender ?? null,
      JSON.stringify(order.route ?? []),
    ]
  );

  return { transactionId, metadataId: meta!.id, flags };
}

/**
 * Second pass over orders parked awaiting a card statement.
 *
 * Run after each statement import. For every pending order it retries the
 * reconciliation; once the card leg appears, any remainder above it was paid in
 * credits and becomes a transaction at that point. Orders whose card leg covers
 * the whole total resolve to "fully card" and correctly create nothing.
 *
 * Idempotent: a resolved row gets reconciled_at set and is never revisited.
 */
export async function reconcilePendingOrders(): Promise<{
  examined: number;
  resolved: number;
  created: number;
}> {
  const pendingRows = await queryRaw<{
    id: number;
    order_reference: string;
    amount: string;
    transaction_date: string;
    merchant_normalized: string;
    card_last4: string | null;
    currency: string | null;
    platform: ParsedOrder["platform"] | null;
  }>(
    `SELECT id, order_reference, amount::text, transaction_date::text,
            merchant_normalized, card_last4, currency, platform
       FROM expense_metadata
      WHERE transaction_id IS NULL
        AND reconciled_at IS NULL
        AND card_last4 IS NOT NULL`
  );

  let resolved = 0;
  let created = 0;

  for (const row of pendingRows) {
    const total = Number(row.amount);
    const probe: ParsedOrder = {
      order_reference: row.order_reference,
      platform: row.platform ?? "doordash",
      merchant_name: row.merchant_normalized,
      order_datetime: `${row.transaction_date}T00:00:00Z`,
      currency: row.currency || "AUD",
      payment: { credits_amount: null, card_amount: null, card_last4: row.card_last4, ambiguous: true },
      totals: {
        subtotal: null, taxes: null, delivery_fee: null,
        service_fee: null, tip: null, discounts: null, total_charged: total,
      },
      line_items: [],
      route: [],
      is_family: false,
      flags: [],
    };

    const { cardAmount, matchedTransactionId } = await reconcileCardLeg(probe);
    if (cardAmount === null) continue; // statement still hasn't arrived

    const remainder = Number((total - cardAmount).toFixed(2));
    let txnId: number | null = null;

    if (remainder > 0.02 && row.transaction_date >= CUTOVER_DATE) {
      // Category from the merchant, never hardcoded. Hardcoding 'dining' here
      // silently misfiled every grocery order that arrived with an unstated
      // split — reintroducing, through the deferred path, exactly the
      // misfiling resolveCategory() exists to prevent.
      const category = resolveCategory({
        merchant_name: row.merchant_normalized,
        platform: "doordash",
      } as ParsedOrder);

      const txn = await queryRow<{ id: number }>(
        `INSERT INTO transactions (
           transaction_date, description, amount, amount_aud, category,
           payment_method, merchant_name, merchant_normalized, transaction_type, owner_id
         ) VALUES ($1,$2,$3,$3,$5,'credits',$4,$4,'debit',$6)
         RETURNING id`,
        [row.transaction_date, orderDescription(row.platform ?? "doordash", row.merchant_normalized), remainder, row.merchant_normalized, category, DEFAULT_OWNER_ID]
      );
      txnId = txn!.id;
      created++;
    }

    await queryRaw(
      `UPDATE expense_metadata
          SET transaction_id = COALESCE($2, transaction_id),
              matched_transaction_id = $4,
              reconciled_at = NOW(),
              flags = flags || $3::jsonb
        WHERE id = $1`,
      [row.id, txnId, JSON.stringify([`card_leg_${cardAmount.toFixed(2)}`]), matchedTransactionId]
    );
    resolved++;
  }

  return { examined: pendingRows.length, resolved, created };
}

/**
 * Category from the merchant.
 *
 * The spec's Correction 1 said never default to `dining`, because ~19% of the
 * corpus is groceries and a blanket dining default misfiles a fifth of orders.
 * That reasoning is right about groceries and wrong about the remedy: the
 * earlier implementation sent everything unrecognised to `other`, and since it
 * knew six merchants, that meant Carl's Jr, Taco Bell, Chilli India, Oporto,
 * Hungry Jacks, Schnitz, Subway, Souvlaki GR and the rest all landed in
 * `other` — worse than the problem it avoided.
 *
 * Deliberate reversal: grocery merchants are a closed, enumerable set;
 * restaurants are an open one. So match groceries explicitly and let the
 * residual be `dining`, which is what a delivery order otherwise is. A
 * misfiled grocer is one rule away from fixed; a corpus in `other` is not.
 */
export function resolveCategory(order: ParsedOrder): string {
  if (order.platform === "uber") return "transport";
  const m = order.merchant_name.toLowerCase();
  if (/woolworths|aldi|coles|glomark|keells|cargills|iga|costco/.test(m)) return "groceries";
  return "dining";
}

/**
 * Applies a refund / total-adjustment notice to an already-ingested order.
 *
 * The order it amends is matched by the UUID Uber reuses across the original
 * receipt and the amendment. The recorded transaction is reduced to the new
 * total rather than a compensating negative row being added: the order is one
 * event and its cost changed, and a second row would misreport both the meal
 * count and the merchant's spend.
 *
 * If the original has not been ingested (amendment arrived first, or the
 * receipt predates the cutover), nothing is invented — the amendment is
 * recorded as unmatched for a later pass.
 */
export async function applyOrderAmendment(a: {
  order_reference: string | null;
  new_total: number;
  refund_amount: number | null;
  previous_total: number | null;
  order_datetime: string;
  messageId: string;
}): Promise<{ matched: boolean; transactionId: number | null; adjusted: number | null }> {
  if (!a.order_reference) return { matched: false, transactionId: null, adjusted: null };

  const meta = await queryRow<{ id: number; transaction_id: number | null; amount: string }>(
    `SELECT id, transaction_id, amount::text FROM expense_metadata
      WHERE source = 'email' AND order_reference = $1`,
    [a.order_reference]
  );
  if (!meta) return { matched: false, transactionId: null, adjusted: null };

  await queryRaw(
    `UPDATE expense_metadata
        SET amount = $2,
            flags = flags || $3::jsonb
      WHERE id = $1`,
    [
      meta.id,
      a.new_total,
      JSON.stringify([
        `amended_from_${Number(meta.amount).toFixed(2)}`,
        a.refund_amount !== null ? `refund_${a.refund_amount.toFixed(2)}` : "amended",
      ]),
    ]
  );

  if (meta.transaction_id === null) {
    // Card-settled or still pending: no transaction of ours to reduce. The
    // refund will show on the statement in its own right.
    return { matched: true, transactionId: null, adjusted: null };
  }

  await queryRaw(
    `UPDATE transactions
        SET amount = $2,
            amount_aud = CASE WHEN foreign_currency_code IS NULL THEN $2 ELSE amount_aud END
      WHERE id = $1`,
    [meta.transaction_id, a.new_total]
  );

  return { matched: true, transactionId: meta.transaction_id, adjusted: a.new_total };
}