finance-app/src/lib/analytics-sql.ts

// Shared SQL fragments for analytics queries, so spend/income semantics stay
// identical across routes.
//
// Three rules every analytics query must follow:
//
// 1. Join `statements` with LEFT JOIN and scope on COALESCE(t.owner_id, s.owner_id).
//    An INNER JOIN silently drops every manual/CSV transaction (statement_id IS
//    NULL) — which is most of the reconciliation and cash-spend data.
// 2. Exclude `transfers` and `investment` from spend. Once bank statements are
//    imported, a credit-card payment appears twice: once as a debit leaving the
//    bank account and again as the underlying purchases on the card statement.
//    Categorising the money movement as `transfers` and excluding it here is what
//    stops the double count. Investments are a balance-sheet move, not spend.
// 3. Apply EXCLUDE_RECONCILED_SOURCE to every row-level query. The transaction
//    queries have always done this (`queries.ts`); analytics never did, which is
//    the other half of the same double count.

/**
 * `YYYY-MM-DD` for a Date, read in local time.
 *
 * `toISOString().slice(0, 10)` is the obvious thing and it is wrong here: these
 * are calendar boundaries built with `new Date(y, m, 1)`, which is local
 * midnight. In any timezone east of UTC that converts to the *previous* day, so
 * every window silently started and ended a day early — visible once the fees
 * endpoint began reporting the range it had used ("2026-04-30" for a window
 * meant to open on 1 May).
 */
export function toDateStr(d: Date): string {
  return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;
}

/** Owner scoping that works for both statement-linked and manual transactions. */
export const OWNER_SCOPE = `COALESCE(t.owner_id, s.owner_id)`;

/** Join clause to pair with OWNER_SCOPE. */
export const STATEMENTS_JOIN = `LEFT JOIN statements s ON s.id = t.statement_id`;

/**
 * Drops rows that a different row has replaced. Two distinct cases:
 *
 * **1. The manual/CSV row a statement line superseded** (`reconciled_with_id`).
 *
 * Reconciliation keeps both rows: the manual one the user entered and the
 * statement line it turned out to be. Only the statement line should count, or
 * the same purchase is spent twice. `queries.ts` has always applied this; the
 * analytics routes did not, so every reconciled row was double-counted in the
 * category totals, movers, Pareto, and merchant rankings.
 *
 * Scoped to `statement_id IS NULL` deliberately: the *source* row is the manual
 * one. A statement line pointing at something else is the survivor, not the
 * duplicate.
 *
 * Order-receipt rows (`payment_method = 'credits'`) are unaffected — they are
 * inserted with `reconciled_with_id` NULL and are held out of the reconcile
 * queue by `needsCardMatch()`, so nothing ever sets it. If one is reconciled by
 * hand against a card line, this is what stops it double-counting.
 *
 * **2. The statement row imported twice** (`superseded_by_id`, migration 0023).
 *
 * When two statements for one account bill overlapping periods, every
 * transaction in the overlap arrives twice — 31 pairs on ANZ `4085-56264`, from
 * statements 107/142/143. Case 1 cannot express this: its predicate is scoped
 * to `statement_id IS NULL` on purpose, and here BOTH rows are statement lines.
 *
 * The superseded row is excluded rather than deleted because every child of
 * `transactions` cascades on delete, and the curation is not reliably on the
 * surviving side.
 *
 * Adding it to this fragment rather than making a new one is deliberate: every
 * query that already asks "count each purchase once" now excludes both kinds
 * without being edited. Anything summing transactions without this fragment
 * still double-counts — that is the same gap that let reconciled rows into the
 * analytics routes in the first place.
 */
export const EXCLUDE_RECONCILED_SOURCE = `NOT (t.statement_id IS NULL AND t.reconciled_with_id IS NOT NULL)
  AND t.superseded_by_id IS NULL`;

/**
 * The currency `t.amount` is actually denominated in.
 *
 * Two different conventions meet here and the COALESCE order is what keeps them
 * apart:
 *
 *  - A statement row is denominated in its statement's currency. If that row is
 *    an overseas purchase on an AUD statement, `amount` is still AUD and
 *    `foreign_currency_code` merely records what was originally charged — so
 *    `s.currency` must win.
 *  - An order-receipt row has no statement. There, `amount` IS the native
 *    figure and `foreign_currency_code` names it (order-ingestion.ts leaves
 *    `amount_aud` NULL rather than asserting an FX rate it does not have).
 *
 * Reading `s.currency` alone labels every foreign order row as AUD.
 * `s` must be the statements alias in scope.
 */
export const NATIVE_CURRENCY = `COALESCE(s.currency, t.foreign_currency_code, 'AUD')`;

/**
 * True when a row's AUD value is unknown: it is denominated in something other
 * than AUD and carries no converted figure.
 *
 * Every settlement total uses `COALESCE(t.amount_aud, t.amount)`, which for such
 * a row silently nets a foreign figure against AUD ones. Rather than drop the
 * row (which changes a balance with no trace) or convert it (with no rate),
 * count these and let the UI say the balance is incomplete.
 */
export const AMOUNT_UNCONVERTED = `(t.amount_aud IS NULL AND ${NATIVE_CURRENCY} <> 'AUD')`;

/** Transaction types that represent money going out. */
export const SPEND_TYPES = `('debit', 'fee', 'interest')`;

/**
 * Rows that count towards spend.
 *
 * The `interest_amount IS NOT NULL` arm is for loan repayments: when a lender
 * itemises principal and interest on a single repayment row, that row is usually
 * typed 'payment' (money reducing the loan balance) and would otherwise be
 * skipped — but its interest portion is real spend.
 */
export const SPEND_ROWS = `(t.transaction_type IN ('debit', 'fee', 'interest') OR t.interest_amount IS NOT NULL)`;

/**
 * The amount of a row that counts as spend, before split adjustment.
 *
 * For an itemised loan repayment only the interest portion is an expense; the
 * principal builds equity and is a balance-sheet move, not spending.
 */
export const SPEND_BASE = `CASE WHEN t.interest_amount IS NOT NULL THEN t.interest_amount ELSE COALESCE(t.amount_aud, t.amount) END`;

/** Effective category, honouring overrides. Never NULL. */
export const EFFECTIVE_CATEGORY = `COALESCE(o.category_override, t.category, 'other')`;

/**
 * Rows whose cost is partly mine: ones I paid for, plus ones someone else paid
 * for that I hold a split on.
 *
 * Every spend analytic used to gate on `${OWNER_SCOPE} = $1` alone, with
 * `mySplitOf` scaling *within* that gate. Ownership was therefore a
 * precondition for an expense being mine — which is backwards for a shared
 * household. Half a grocery shop is my expense whether the card was mine or
 * Sonu's; who owns the row says whose *account* the money left, a different
 * question. It is the same principle the `settled` rule already states: my
 * share is my expense regardless of who fronted it or whether it has since
 * been repaid.
 *
 * Measured on 2026-08-15 the gate hid 167 rows / $3,210.91 of my own share
 * (Jan–Jul 2026) from monthly, daily, merchants, subscriptions, fees and the
 * budget page, while `getParticipantBalances` booked the matching debt
 * correctly — so the app could tell you that you owed Sonu for a shop while
 * insisting you had not spent anything on it. Worst month was April 2026
 * (+$1,354.83), the Europe trips, where she paid for a lot.
 *
 * That figure is net of `EXCLUDE_NON_BUDGET_TAGS`: a further $456.50 of
 * non-owned share in July is `family`-tagged and correctly stays out. Counting
 * on transaction_type alone gives $3,667.41 and is the wrong comparison.
 *
 * Requires `transaction_splits ts` joined on `ts.participant_id =
 * <participant>` and ${STATEMENTS_JOIN}. That join cannot multiply rows:
 * `(transaction_id, participant_id)` is UNIQUE.
 *
 * Pair it with `myShare`, which returns 0 for a non-owned row with no split —
 * so admitting a row here can never add more than the share actually held.
 */
export const MY_SPEND_SCOPE = (participant = "$1") =>
  `(${OWNER_SCOPE} = ${participant} OR ts.participant_id IS NOT NULL)`;

/**
 * My percentage share of a transaction, 0-100.
 *
 * On a row I paid for, resolution order:
 *   1. An explicit `transaction_splits` row for me.
 *   2. The `my_share_percent` override.
 *   3. Whatever is left after everyone else's shares.
 *
 * Step 3 is the one that matters. Assuming 100% when no split row exists for me
 * is wrong whenever a transaction is allocated entirely to someone else — I paid,
 * they owe all of it, and there is no row for me to find. Those rows would
 * otherwise land in my spend at full value.
 *
 * On a row SOMEONE ELSE paid for, only an explicit split row counts, and its
 * absence means 0. Both of the other two steps are meaningless there and one of
 * them is actively wrong:
 *
 *  - `my_share_percent` is a single column on `transaction_overrides` with no
 *    participant scoping, writable by anyone who can see the row. "My" can only
 *    coherently mean the owner's. Reading it as the viewer's share on someone
 *    else's transaction would be a guess, and all 402 rows carrying one today
 *    are owner-side, so there is nothing to be gained by guessing.
 *  - `100 - everyone else` is the *payer's* remainder by definition. Applied to
 *    a row I did not pay for it would hand me the payer's share on top of my
 *    own.
 *
 * Requires `transaction_splits ts` joined on `ts.participant_id = <participant>`,
 * `transaction_overrides o` joined on the transaction, and ${STATEMENTS_JOIN}.
 */
export const myShare = (participant = "$1") => `CASE
    WHEN ${OWNER_SCOPE} = ${participant} THEN COALESCE(
      ts.share_percent,
      o.my_share_percent,
      100 - COALESCE((
        SELECT SUM(x.share_percent) FROM transaction_splits x
        WHERE x.transaction_id = t.id AND x.participant_id <> ${participant}
      ), 0)
    )
    ELSE COALESCE(ts.share_percent, 0)
  END`;

/** `base` scaled to my share. Use for every per-user spend total. */
export const mySplitOf = (base: string, participant = "$1") =>
  `((${base}) * ${myShare(participant)} / 100)`;

export const NON_BUDGET_TAGS = ['family'];

export const EXCLUDE_NON_BUDGET_TAGS = `NOT EXISTS (
  SELECT 1 FROM transaction_tags tt JOIN tags tg ON tg.id = tt.tag_id
  WHERE tt.transaction_id = t.id AND tg.name = ANY(ARRAY['family'])
)`;

/**
 * Predicate excluding categories and tags that are not spend.
 * The COALESCE matters: a bare `category NOT IN (...)` evaluates to NULL for
 * uncategorised rows, which silently drops them from spend totals.
 */
export const EXCLUDE_NON_SPEND = `${EFFECTIVE_CATEGORY} NOT IN ('transfers', 'investment', 'income') AND ${EXCLUDE_NON_BUDGET_TAGS}`;

/**
 * Rows that count towards NET spend — outgoings plus the refunds that cancel
 * them. Pair with SPEND_SIGNED, which carries the direction.
 */
export const NET_SPEND_ROWS = `(${SPEND_ROWS} OR t.transaction_type IN ('refund', 'credit'))`;

/**
 * SPEND_BASE, signed: refunds and credits come back as negatives so they cancel
 * the original purchase.
 *
 * Without this a refund is counted nowhere. It is excluded from spend by type
 * and only counted as income if categorised 'income', which a refund is not —
 * so it falls through both and never reduces anything. A $2,888.92 Expedia
 * purchase refunded in full eight days later still read as $2,888.92 of spend.
 */
export const SPEND_SIGNED = `CASE
  WHEN t.transaction_type IN ('refund', 'credit') THEN -(${SPEND_BASE})
  ELSE (${SPEND_BASE})
END`;

/**
 * The investment line, signed: withdrawals come back as negatives so they net
 * against contributions.
 *
 * A withdrawal from a fund is a disinvestment — units converted back to cash,
 * net worth unchanged. Summed unsigned it read as *more* money invested: March
 * 2026 showed $38,615.34 of investing in a month that was net -$11,384.66,
 * because a $25,000 Raiz withdrawal was added to a $8,563.80 IBKR deposit
 * instead of cancelling it. Overstated by $50,000 in that month alone — each
 * credit costs twice, once for being added and once for not being subtracted.
 *
 * Filing withdrawals as `income` instead is the other tempting answer and is
 * worse: it books an asset disposal as earnings and feeds the same figure into
 * `net = income - spent - investments` with a flattering sign. Same reason the
 * Up item sales in Known Gaps do not belong on the income line.
 *
 * Caveat this cannot resolve: part of a withdrawal genuinely is income — the
 * capital gain. The bank descriptor is a single gross figure with no cost base,
 * so it cannot be decomposed here. Netting tracks cash committed against cash
 * returned and leaves the gain for holdings data to surface; it does not claim
 * the gain is zero.
 */
export const INVESTMENT_SIGNED = `CASE
  WHEN t.transaction_type IN ('refund', 'credit') THEN -COALESCE(t.amount_aud, t.amount)
  ELSE COALESCE(t.amount_aud, t.amount)
END`;

/**
 * A split that still counts towards what someone owes.
 *
 * This is the line between the two questions the same table answers, and
 * conflating them is what made three different "owed" figures disagree:
 *
 *  - **Owed** — what is still outstanding between two people. Must apply this.
 *  - **Spend** — what a purchase cost me. Must NOT apply this.
 *
 * A settled split is still a real expense: my half of a 2025 grocery shop is my
 * spend whether or not the other half was ever repaid. Filtering settled rows
 * out of `myShare`/`mySplitOf` would re-inflate exactly the figures that
 * importing settled history exists to correct.
 *
 * `settled` marks obligations discharged OUTSIDE this app — imported
 * SplitMyExpenses history, whose repayments happened on a platform we no longer
 * run and which therefore has no `split_payments` row here.
 *
 * A live obligation is NOT settled by flipping this. It is settled by recording
 * the payment, and the balance nets to zero on its own. Doing both would
 * subtract the settlement twice — the splits leave the sum AND the payment is
 * deducted — driving the balance negative by the amount repaid. So there is
 * deliberately no "mark settled" action anywhere: settling up is recording a
 * payment, and this column is only ever written by the historical import.
 *
 * **The date is the real guard, and the flag is only a refinement of it.**
 * Nothing before SPLIT_CUTOVER can be owed, because carryover transaction 2348
 * already carries the entire pre-cutover balance as a single figure. A split on
 * an older transaction is therefore free to describe *how an expense was shared*
 * — which is what stops it inflating spend — without ever asserting a debt.
 *
 * That separation is what makes splitting history safe. Before it, the only
 * thing keeping $37,233.28 of paid debt out of the balances was a boolean that
 * any delete-and-recreate write path silently reset to false. Now losing the
 * flag on a pre-cutover row costs nothing: the date still excludes it. The flag
 * matters only for rows on or after the cutover, where it marks the handful
 * settled outside this app.
 *
 * The boundary is inclusive because transaction 2348 is itself dated
 * 2026-01-09 — an exclusive bound would drop the carryover and with it the
 * entire pre-cutover balance.
 *
 * Assumes the `transaction_splits` alias is `ts` and `transactions` is `t`.
 */
export const SPLIT_CUTOVER = "2026-01-09";

export const ACTIVE_OBLIGATION = `ts.settled = false
  AND t.transaction_date >= '${SPLIT_CUTOVER}'`;

/**
 * The tab a split belongs to: its transaction's trip, else the household.
 *
 * Membership already lives on `transaction_overrides.trip_id`, so this is a
 * read of existing data rather than a new grouping key. Assumes an
 * `transaction_overrides` alias `o` is joined (LEFT — a transaction with no
 * override row has no trip, which is the common case and means household).
 */
export const SPLIT_SCOPE = `o.trip_id`;