finance-app/src/lib/wallet-ingestion.ts

import { queryRow, withTransaction, type Queryable } from "@/lib/db";
import { DEFAULT_OWNER_ID } from "@/lib/order-ingestion";
import { findWalletAdoptableTransaction } from "@/lib/auto-reconcile";
import {
  parseWestpacDescriptor,
  parseWalletSubtitle,
  currencyFromAmountText,
} from "@/lib/wallet-descriptor";

/**
 * Wallet notifications -> transactions.
 *
 * A transaction is created for EVERY tap (DECISIONS.md ING-11, 2026-08-22),
 * because overseas trips are where live spend matters most: the statement lag
 * is the same 6-35 days abroad but knowing what you have spent matters far more
 * in an unfamiliar currency.
 *
 * SEQUENCING IS THE WHOLE DESIGN. Wallet is the FIRST signal, not a late one --
 * measured on txn 6858, the notification arrived 35h25m before anything created
 * a transaction. So this creates the row, the receipt enriches it
 * (receipt-ingestion.ts adopts it), and the statement confirms it via the
 * reconcile queue, which a wallet row enters unaided: it is statementless with
 * payment_method 'card'. Nothing here needs to know about statements.
 *
 * But that sequence is a TENDENCY, not a guarantee -- these first 21 rows were
 * themselves backfilled days after the taps -- so this also adopts a row a
 * receipt got to first (findWalletAdoptableTransaction), and a stored-value
 * instrument is written as 'credits' so it never enters a queue no statement
 * line will ever resolve.
 */

export type WalletEvent = {
  /** capture.wallet_events.id -- the idempotency key. */
  event_id: number;
  /** 'transaction' | 'not_transaction' | 'unsure' from the interpreter. */
  kind: string;
  direction?: string | null;
  amount: number | null;
  /** The exact substring the model read the amount from, e.g. "A$5,103.49". */
  amount_text?: string | null;
  currency?: string | null;
  merchant?: string | null;
  card_tail?: string | null;
  issuer?: string | null;
  location?: string | null;
  notif_app?: string | null;
  /** The notification's title line, e.g. "Visa Gift Card" or "Payment". */
  notif_title?: string | null;
  notif_body?: string | null;
  notif_subtitle?: string | null;
  occurred_at?: string | null;
};

export type WalletIngestResult = {
  transactionId: number | null;
  metadataId: number | null;
  skipped?:
    | "not_a_transaction" | "already_ingested" | "no_amount" | "merged_pair" | "adopted"
    | "account_notice" | "stored_value_drawdown";
  flags: string[];
};

/**
 * How far apart the two halves of ONE tap can arrive. Measured 2-8 seconds on
 * every captured pair; ten minutes leaves room for a phone that was offline,
 * and is still nowhere near the +/-1 DAY the matcher used to allow -- which
 * folded the UNICEF $50 donation and an unrelated $50 bar charge 17 hours later
 * into one row (event 275 into txn 8091, 2026-10-01).
 */
const PAIR_WINDOW_MINUTES = 10;

/** The ledger is dated the way the bank statements are: Melbourne local. */
const LEDGER_TZ = "Australia/Melbourne";

/**
 * The calendar date of an instant, in LEDGER_TZ.
 *
 * Slicing the ISO string took whatever offset n8n serialised -- UTC in
 * practice -- so a Melbourne-evening tap landed on the previous day (28 of the
 * first 134 rows). Statements, which this row is reconciled against, date in
 * Australian local time.
 */
export function ledgerDate(occurredAt: string | null | undefined, now: Date = new Date()): string {
  const d = occurredAt ? new Date(occurredAt) : now;
  const at = Number.isNaN(d.getTime()) ? now : d;
  // en-CA formats as YYYY-MM-DD.
  return new Intl.DateTimeFormat("en-CA", {
    timeZone: LEDGER_TZ, year: "numeric", month: "2-digit", day: "2-digit",
  }).format(at);
}

/**
 * A ledger-derived AUD rate for a currency: the median of what the bank
 * actually charged divided by the native figure, over statement rows.
 *
 * Same source order_transaction_bridge uses for its plausibility guard. This is
 * only ever a PROVISIONAL value -- transactions.amount is NOT NULL and always
 * AUD, and a notification cannot know the AUD figure because the bank sets the
 * rate at settlement. The statement overwrites it at reconcile time.
 * Returns null when there is not enough history to be worth anything.
 */
async function medianRateToAud(code: string): Promise<number | null> {
  const row = await queryRow<{ rate: string | null }>(
    `SELECT percentile_cont(0.5) WITHIN GROUP (
              ORDER BY amount / NULLIF(foreign_currency_amount, 0)
            )::text AS rate
       FROM transactions
      WHERE foreign_currency_code = $1
        AND foreign_currency_amount > 0
        AND superseded_by_id IS NULL
        -- Statement rows only: a wallet row's amount is itself a provisional
        -- guess, and one stored before this filter (txn 7316, 672006 VND at a
        -- 1:1 "rate") would otherwise feed the next estimate.
        AND statement_id IS NOT NULL
      HAVING count(*) >= 5`,
    [code]
  );
  const rate = row?.rate ? Number(row.rate) : null;
  return rate && Number.isFinite(rate) && rate > 0 ? rate : null;
}

/**
 * The ECB reference rate for the day, via Frankfurter (free, no key; the same
 * source the statement processor converts with).
 *
 * The fallback when the ledger has no history for a currency. Hong Kong was the
 * first trip to need it: no HKD statement row existed, so every HKD tap stored
 * A$0.00 and the trip read as free until the statement landed weeks later. The
 * ledger median is still preferred -- it carries the card's ~3% foreign fee,
 * which the ECB mid-rate does not -- so this one is flagged as such.
 *
 * Any failure is null, never a throw: no rate is a state this lane already
 * handles (a flagged 0), and a slow FX API must not lose the purchase.
 */
async function ecbRateToAud(code: string, date: string): Promise<number | null> {
  try {
    const res = await fetch(
      `https://api.frankfurter.dev/v1/${date}?from=${encodeURIComponent(code)}&to=AUD`,
      { signal: AbortSignal.timeout(5000) }
    );
    if (!res.ok) return null;
    const body = (await res.json()) as { rates?: { AUD?: number } };
    const rate = body?.rates?.AUD;
    return typeof rate === "number" && Number.isFinite(rate) && rate > 0 ? rate : null;
  } catch {
    return null;
  }
}

/**
 * The other half of a paired notification, if it has already been ingested.
 *
 * One tap emits TWO notifications ~2-8s apart: the issuer app (which has the
 * card tail) and Apple Wallet (which has the better merchant and the suburb).
 * Neither alone is complete, so they must be MERGED, never deduped -- dropping
 * either loses a field nothing else supplies. The notifications share no
 * identifier, so a pair is recognised by what a pair IS: the same amount,
 * minutes apart, from two different apps. Two notifications from one app are
 * two charges, however close together.
 */
async function findPairedRow(
  db: Queryable,
  amount: number,
  notifiedAt: string,
  notifApp: string | null,
  fx: { amount: number; code: string } | null = null
) {
  // A foreign row pairs on its native pair: its `amount` is a provisional
  // conversion (or a 0 placeholder), so two unrelated foreign taps could share it.
  const rows = await db.queryRaw<{ id: number; meta_id: number }>(
    `SELECT t.id, em.id AS meta_id
       FROM transactions t
       JOIN expense_metadata em ON em.transaction_id = t.id
      WHERE t.source = 'wallet'
        AND t.statement_id IS NULL
        AND t.superseded_by_id IS NULL
        AND CASE WHEN $4::text IS NULL
                 THEN t.amount = $1::numeric AND t.foreign_currency_code IS NULL
                 ELSE t.foreign_currency_code = $4::text
                      AND t.foreign_currency_amount = $5::numeric END
        AND em.notified_at BETWEEN $2::timestamptz - make_interval(mins => $3::int)
                               AND $2::timestamptz + make_interval(mins => $3::int)
        AND em.notif_app IS DISTINCT FROM $6::text
      LIMIT 2`,
    [amount, notifiedAt, PAIR_WINDOW_MINUTES, fx?.code ?? null, fx?.amount ?? null, notifApp]
  );
  return rows.length === 1 ? rows[0] : null;
}

/**
 * A stored-value instrument: a gift card, a prepaid travel card, a store card.
 *
 * It matters because payment_method decides queue membership. 'card' means "a
 * statement line is coming"; findAutoMatches and the reconcile queue both
 * exclude 'credits' and 'cash' for exactly that reason. A $103.30 Woolworths
 * tap on a Visa gift card written as 'card' parks in the queue forever, being
 * offered as an unmatched row against statements that can never contain it --
 * three of the first 21 wallet rows (events 80, 92, 101) were gift-card taps.
 *
 * The evidence is the notification TITLE, which is where Apple Wallet prints
 * the instrument's own name ("Visa Gift Card"); the interpreter reduces that to
 * issuer "Visa", which a plain Visa debit card also produces, so the title is
 * the only place the distinction survives. Read as a prior, not a contract: a
 * miss leaves today's behaviour, which is the queue.
 */
const STORED_VALUE = /gift\s*card|prepaid|travel\s*(?:money|card)|store\s*card/i;

/**
 * A transit card reporting a drawdown of its own balance ("HKD 7.40 for Train"
 * / "Your new balance is HKD 169.50.").
 *
 * Unlike a gift card, a transit card is loaded FROM a bank card -- that load is
 * on the statement and is the spend. Each fare is that money being used, so
 * booking it again counts it twice. The interpreter already called 11 of 13
 * Octopus fares not_transaction; the two it did not (events 191, 200) became
 * rows, so the outcome depended on the model's mood. This makes it a rule. The
 * body wording is the signal because it is what every such card says; the
 * issuer name is not (Octopus, Opal, myki, Suica each name themselves).
 */
const BALANCE_DRAWDOWN = /\byour (?:new|current) balance is\b/i;

/**
 * Account notices that state an amount without money moving: "Your Smart Access
 * 1910 is overdrawn by $21.21. To avoid fees and interest, ..." (event 235)
 * became a $21.21 purchase named "To avoid fees and interest,". The model is
 * right that the text names an amount; it is not a payment.
 */
const ACCOUNT_NOTICE = /\boverdrawn\b|\bto avoid fees\b|\bavailable balance\b|\blow balance\b/i;

/** How an adopted or merged row records which wallet event folded into it. */
const adoptedFlag = (eventId: number) => `wallet_evt:${eventId}`;

export function isStoredValueInstrument(ev: Pick<WalletEvent, "notif_title" | "issuer" | "notif_app">): boolean {
  return [ev.notif_title, ev.issuer, ev.notif_app].some((v) => !!v && STORED_VALUE.test(v));
}

/** Already landed: a row this lane wrote, or one it adopted or merged into. */
async function alreadyIngested(db: Pick<Queryable, "queryRow">, eventId: number) {
  return db.queryRow<{ id: number }>(
    `SELECT id FROM transactions WHERE source = 'wallet' AND source_ref = $1
      UNION ALL
     SELECT em.transaction_id AS id FROM expense_metadata em
      JOIN transactions t ON t.id = em.transaction_id
      WHERE em.flags @> $2::jsonb AND t.superseded_by_id IS NULL
      LIMIT 1`,
    [`evt:${eventId}`, JSON.stringify([adoptedFlag(eventId)])]
  );
}

export async function processWalletIngestion(ev: WalletEvent): Promise<WalletIngestResult> {
  const flags: string[] = [];

  // An open-world classifier feeding a closed-world writer drops the unknown
  // and calls it a refusal (ING-14). `unsure` is a real answer and must not
  // silently become a transaction.
  if (ev.kind !== "transaction") {
    return { transactionId: null, metadataId: null, skipped: "not_a_transaction", flags };
  }
  if (ev.amount === null || !Number.isFinite(ev.amount) || ev.amount <= 0) {
    return { transactionId: null, metadataId: null, skipped: "no_amount", flags };
  }
  const text = [ev.notif_subtitle, ev.notif_body].filter(Boolean).join("\n");
  if (ACCOUNT_NOTICE.test(text)) {
    return { transactionId: null, metadataId: null, skipped: "account_notice", flags };
  }
  if (BALANCE_DRAWDOWN.test(ev.notif_body ?? "")) {
    return { transactionId: null, metadataId: null, skipped: "stored_value_drawdown", flags };
  }

  // Two places this event may already have landed: a row this lane wrote (which
  // carries the ref natively), or a row it ADOPTED or MERGED into, whose
  // source_ref belongs to whoever created it -- marked in the metadata flags
  // rather than by overwriting a field another lane owns. Both must be checked,
  // because bumping the interpreter's prompt_version deliberately re-interprets
  // and re-posts every event ever captured -- that is the replay mechanism, and
  // it must not mint a second row for a purchase that has one. Checked here
  // without the lock to skip the FX lookup on a replay, and again under it.
  const seen = await alreadyIngested({ queryRow }, ev.event_id);
  if (seen) return { transactionId: seen.id, metadataId: null, skipped: "already_ingested", flags };

  const sourceRef = `evt:${ev.event_id}`;
  const notifiedAt = ev.occurred_at ?? new Date().toISOString();
  const date = ledgerDate(ev.occurred_at);

  // Deterministic parsing of the positional fields. The model decides what the
  // amount and merchant ARE; it does not get to parse a fixed-width block.
  // Only Westpac writes that block: run over a CommBank overdraft notice it
  // produced the merchant "To avoid fees and interest," (event 235).
  const desc = /westpac/i.test(ev.notif_app ?? "")
    ? parseWestpacDescriptor(ev.notif_body)
    : { merchant: null, city: null, country: null };
  const sub = parseWalletSubtitle(ev.notif_subtitle);
  const merchant = sub.merchant ?? ev.merchant ?? desc.merchant ?? "Card purchase";
  // Two sources for a place, and they are not equal. The Apple Wallet subtitle
  // gives "Wyndham Vale, VIC"; the issuer's fixed-width descriptor gives
  // "WYNDHAM VAL AU" -- a 12-character truncation in caps. The subtitle wins
  // outright, which matters for the pair merge below: whichever half arrives
  // second must be able to REPLACE a descriptor-derived place, or the result
  // depends on arrival order.
  const subtitleLocation = sub.location ?? null;
  // Precedence: the Wallet subtitle, then the DETERMINISTIC descriptor parse,
  // and only then the model's own answer. Getting this order wrong defeats the
  // point of parsing positionally at all -- the model returned the mangled
  // "SAN FRANCISOUS" and the padded "MELBOURNE   AU" for exactly these rows,
  // which is what wallet-descriptor.ts exists to correct.
  const descriptorLocation = desc.city && desc.country
    ? `${desc.city} ${desc.country}`
    : desc.city;
  const location = subtitleLocation ?? descriptorLocation ?? ev.location ?? null;

  // Currency: only ever what the notification actually states. A bare "$" is
  // recorded as unknown, never assumed AUD -- harmless domestically, a 23x
  // error on a THB tap abroad.
  const stated = currencyFromAmountText(ev.amount_text)
    ?? (ev.currency && /^[A-Z]{3}$/.test(ev.currency) ? ev.currency : null);
  if (!stated) flags.push("currency_unverified");
  if (desc.country && desc.country !== "AU") flags.push(`foreign_country_${desc.country}`);

  // Foreign: the native figure is what both sides can agree on exactly, so it
  // is what gets stored. `amount` still needs a value (NOT NULL) and takes a
  // provisional conversion, flagged, which the statement overwrites. Resolved
  // BEFORE the lock: the ECB fallback is a network call, and the lock
  // serialises every wallet post behind whatever runs inside it.
  let amountAud = ev.amount;
  let fxAmount: number | null = null;
  let fxCode: string | null = null;
  if (stated && stated !== "AUD") {
    fxAmount = ev.amount;
    fxCode = stated;
    const ledgerRate = await medianRateToAud(stated);
    const rate = ledgerRate ?? (await ecbRateToAud(stated, date));
    if (rate) {
      amountAud = Number((ev.amount * rate).toFixed(2));
      flags.push("aud_provisional");
      if (!ledgerRate) flags.push("rate_ecb");
    } else {
      // No rate from either source. The native figure must NOT fall through as
      // AUD -- 672006 VND once landed as A$672,006. Zero is a flagged
      // placeholder; the statement supplies the real AUD at reconcile time,
      // which matches on the native pair, not on this column.
      amountAud = 0;
      flags.push("aud_provisional", "no_rate_reference");
    }
  }

  // A gift card has no statement line coming, so it must not enter the
  // reconcile queue (which filters on exactly this column).
  const storedValue = isStoredValueInstrument(ev);
  const paymentMethod = storedValue ? "credits" : "card";
  if (storedValue) flags.push("stored_value_instrument");

  // Everything from the idempotency re-check to the INSERT holds one lock.
  // n8n's HTTP node posts a poll's items CONCURRENTLY, and both halves of a tap
  // usually land in the same 15-minute poll -- so each ran findPairedRow before
  // the other had inserted, neither found its partner, and one tap became two
  // rows: 11 pairs, A$3,841.40, every pair with an identical created_at. A
  // single lock for the whole lane, not one per amount: wallet volume is a few
  // dozen a day, and a coarse lock has no key to get wrong.
  return withTransaction(async (db) => {
    await db.queryRaw(`SELECT pg_advisory_xact_lock(hashtextextended('wallet-ingest', 0))::text`);

    const existing = await alreadyIngested(db, ev.event_id);
    if (existing) {
      return { transactionId: existing.id, metadataId: null, skipped: "already_ingested" as const, flags };
    }

    // Merge the paired notification rather than creating a second row for one
    // tap. The issuer half brings the card tail; the Wallet half brings the
    // merchant and suburb. COALESCE so whichever arrives second only ever fills
    // gaps -- it can never blank a field the first half supplied.
    const pair = await findPairedRow(
      db, amountAud, notifiedAt, ev.notif_app ?? null,
      fxCode && fxAmount !== null ? { amount: fxAmount, code: fxCode } : null
    );
    if (pair) {
      await db.queryRaw(
        `UPDATE transactions
            SET merchant_name       = COALESCE(NULLIF($2,'Card purchase'), merchant_name),
                merchant_normalized = COALESCE(NULLIF($2,'Card purchase'), merchant_normalized),
                -- A subtitle-derived place overwrites; anything else only fills a
                -- gap. Without the $4 branch the merged row depends on which
                -- notification happened to arrive first, and the issuer half
                -- usually wins by a few seconds -- keeping the worse string.
                location            = CASE WHEN $4::boolean THEN COALESCE($3, location)
                                           ELSE COALESCE(location, $3) END
          WHERE id = $1`,
        [pair.id, merchant, location, subtitleLocation !== null]
      );
      // The merged half records itself, or a replay finds nothing and merges
      // it again (or, once the partner is superseded, mints a new row).
      await db.queryRaw(
        `UPDATE expense_metadata
            SET card_last4            = COALESCE(card_last4, $2),
                payment_method_detail = COALESCE(payment_method_detail, $3),
                merchant_normalized   = COALESCE(NULLIF($4,'Card purchase'), merchant_normalized),
                flags                 = COALESCE(flags, '[]'::jsonb) || $5::jsonb
          WHERE id = $1`,
        [pair.meta_id, ev.card_tail ?? null, ev.issuer ?? null, merchant, JSON.stringify([adoptedFlag(ev.event_id)])]
      );
      return { transactionId: pair.id, metadataId: pair.meta_id, skipped: "merged_pair" as const, flags };
    }

    // The receipt got here first. Fold into its row instead of writing a second
    // one for the same purchase (see findWalletAdoptableTransaction). The column
    // list IS the ownership boundary, the mirror of receipt-ingestion.ts's: the
    // receipt read the register tape, so merchant, description, category and the
    // items are its business and are absent here; `location` is wallet's own --
    // nothing else writes a suburb -- and payment_method is filled only when the
    // other lane left it unknown.
    const adopted = await findWalletAdoptableTransaction(DEFAULT_OWNER_ID, amountAud, date, merchant, undefined, db);
    if (adopted) {
      await db.queryRaw(
        `UPDATE transactions
            SET location       = COALESCE($2, location),
                payment_method = COALESCE(payment_method, $3)
          WHERE id = $1`,
        [adopted.id, location, paymentMethod]
      );
      let metadataId = adopted.metadata_id;
      if (metadataId) {
        await db.queryRaw(
          `UPDATE expense_metadata
              SET card_last4            = COALESCE(card_last4, $2),
                  payment_method_detail = COALESCE(payment_method_detail, $3),
                  currency              = COALESCE(currency, $4),
                  flags                 = COALESCE(flags, '[]'::jsonb) || $5::jsonb
            WHERE id = $1`,
          [metadataId, ev.card_tail ?? null, ev.issuer ?? null, stated, JSON.stringify([...flags, adoptedFlag(ev.event_id)])]
        );
      } else {
        // expense_metadata.transaction_id is UNIQUE, so this branch only runs
        // when the adopted row genuinely has none -- and it must produce one, or
        // the card tail and the flag that makes this idempotent have nowhere to
        // live.
        const meta = await db.queryRow<{ id: number }>(
          `INSERT INTO expense_metadata (
             transaction_id, source, source_message_id, merchant_normalized, amount,
             transaction_date, card_last4, currency, payment_method,
             payment_method_detail, line_items, flags, extraction_model,
             notified_at, notif_app
           ) VALUES ($1,'wallet',$2,$3,$4,$5,$6,$7,$8,$9,'[]'::jsonb,$10::jsonb,'wallet-interpret',$11,$12)
           RETURNING id`,
          [
            adopted.id, sourceRef, merchant, ev.amount, date, ev.card_tail ?? null, stated,
            paymentMethod, ev.issuer ?? null, JSON.stringify([...flags, adoptedFlag(ev.event_id)]),
            notifiedAt, ev.notif_app ?? null,
          ]
        );
        metadataId = meta!.id;
      }
      return { transactionId: adopted.id, metadataId, skipped: "adopted" as const, flags };
    }

    const txn = await db.queryRow<{ id: number }>(
      `INSERT INTO transactions (
         transaction_date, description, amount, amount_aud, category, payment_method,
         merchant_name, merchant_normalized, transaction_type, owner_id, location,
         foreign_currency_amount, foreign_currency_code, source, source_ref, row_index
       ) VALUES ($1,$2,$3,$3,'other',$10,$4,$4,'debit',$5,$6,$7,$8,'wallet',$9,0)
       RETURNING id`,
      [date, merchant, amountAud, merchant, DEFAULT_OWNER_ID, location, fxAmount, fxCode, sourceRef, paymentMethod]
    );

    const meta = await db.queryRow<{ id: number }>(
      `INSERT INTO expense_metadata (
         transaction_id, source, source_message_id, merchant_normalized, amount,
         transaction_date, card_last4, currency, payment_method,
         payment_method_detail, line_items, flags, extraction_model,
         notified_at, notif_app
       ) VALUES ($1,'wallet',$2,$3,$4,$5,$6,$7,$8,$9,'[]'::jsonb,$10::jsonb,'wallet-interpret',$11,$12)
       RETURNING id`,
      [
        txn!.id, sourceRef, merchant, ev.amount, date,
        ev.card_tail ?? null, stated, paymentMethod, ev.issuer ?? null, JSON.stringify(flags),
        notifiedAt, ev.notif_app ?? null,
      ]
    );

    return { transactionId: txn!.id, metadataId: meta!.id, flags };
  }, { maxWait: 10_000, timeout: 30_000 });
}