feat(statements+analytics): normalise statement_type; fix analytics scoping

Groundwork for importing bank and loan statements alongside credit cards. statement_type was whatever free text Gemini put in account_type ('Credit Card', 'credit card', 'credit_card', 'Business Card', 'ACCESS ADVANTAGE', 'multi-currency account'). The UI coped only by doing .includes("card"), which breaks as soon as bank and loan statements arrive. - Migration 0013 adds normalize_statement_type() + a BEFORE INSERT/UPDATE trigger and a CHECK constraint over credit_card|transaction|savings|loan| offset|investment|other. The trigger means the N8N workflow keeps working unchanged while it still sends free text. Raw value stays in account_type. Backfilled 99 existing rows. - src/lib/statement-types.ts mirrors the vocabulary for the UI; statements page now filters by the real types and headlines balance vs amount due per type. Analytics were scoped with INNER JOIN statements + s.owner_id, which silently dropped all 180 manual/CSV transactions (statement_id IS NULL) from every report. Switched all six routes to LEFT JOIN + COALESCE(t.owner_id, s.owner_id) via shared fragments in src/lib/analytics-sql.ts, so the transfers/investment exclusion that stops card-payment double counting stays consistent. Also extended that exclusion to trip analytics, which had none. Drive-by: /api/analytics/subscriptions was returning 500 on an unserialisable BigInt from COUNT(*) + 1. Verified against the live DB: monthly spend picks up the previously invisible manual transactions (Apr 9,849.91 -> 14,286.70) and all four analytics endpoints return 200.
2026-07-26 00:00:55 +10:00
parent e0b0fc91e0
commit 25ef504574
12 changed files with 269 additions and 39 deletions
@@ -1,6 +1,7 @@
 import { NextRequest, NextResponse } from "next/server";
 import { getCurrentUser } from "@/lib/auth";
 import { queryRaw } from "@/lib/db";
+import { OWNER_SCOPE, STATEMENTS_JOIN } from "@/lib/analytics-sql";

 export async function GET(req: NextRequest) {
  const user = await getCurrentUser(req);
@@ -45,12 +46,12 @@ export async function GET(req: NextRequest) {
        WHEN o.my_share_percent IS NOT NULL THEN COALESCE(t.amount_aud, t.amount) * o.my_share_percent / 100
        ELSE COALESCE(t.amount_aud, t.amount)
      END::numeric(12,2) AS my_amount,
-      s.bank_name
+      COALESCE(s.bank_name, 'Manual') AS bank_name
    FROM transactions t
    LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
    LEFT JOIN transaction_splits ts ON ts.transaction_id = t.id AND ts.participant_id = $1
-    JOIN statements s ON s.id = t.statement_id
-    WHERE s.owner_id = $1
+    ${STATEMENTS_JOIN}
+    WHERE ${OWNER_SCOPE} = $1
      AND t.transaction_type IN ('fee', 'interest')
    ORDER BY t.transaction_date DESC`,
    [user.id]
@@ -1,6 +1,7 @@
 import { NextRequest, NextResponse } from "next/server";
 import { getCurrentUser } from "@/lib/auth";
 import { queryRaw } from "@/lib/db";
+import { OWNER_SCOPE, STATEMENTS_JOIN } from "@/lib/analytics-sql";

 export async function GET(
  req: NextRequest,
@@ -38,13 +39,13 @@ export async function GET(
      END::numeric(10,2) as my_amount,
      t.transaction_type,
      COALESCE(o.category_override, t.category) as category,
-      s.bank_name,
+      COALESCE(s.bank_name, 'Manual') as bank_name,
      t.statement_id
    FROM transactions t
-    JOIN statements s ON s.id = t.statement_id
+    ${STATEMENTS_JOIN}
    LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
    LEFT JOIN transaction_splits ts ON ts.transaction_id = t.id AND ts.participant_id = $1
-    WHERE s.owner_id = $1
+    WHERE ${OWNER_SCOPE} = $1
      AND t.transaction_type IN ('debit', 'fee', 'interest', 'refund', 'credit')
      AND COALESCE(o.merchant_normalized, t.merchant_normalized, t.merchant_name, t.description) = $2
    ORDER BY t.transaction_date DESC
@@ -1,6 +1,7 @@
 import { NextRequest, NextResponse } from "next/server";
 import { getCurrentUser } from "@/lib/auth";
 import { queryRaw } from "@/lib/db";
+import { OWNER_SCOPE, STATEMENTS_JOIN, EXCLUDE_NON_SPEND } from "@/lib/analytics-sql";

 // Split-adjusted amount helper (positive for spend, negative for refunds)
 const MY_AMOUNT = `CASE WHEN ts.share_percent IS NOT NULL THEN COALESCE(t.amount_aud, t.amount) * ts.share_percent / 100 WHEN o.my_share_percent IS NOT NULL THEN COALESCE(t.amount_aud, t.amount) * o.my_share_percent / 100 ELSE COALESCE(t.amount_aud, t.amount) END`;
@@ -61,13 +62,13 @@ export async function GET(req: NextRequest) {
      MAX(t.transaction_date)::text as last_seen,
      COUNT(DISTINCT TO_CHAR(DATE_TRUNC('month', t.transaction_date::date), 'YYYY-MM'))::int as months_active
    FROM transactions t
-    JOIN statements s ON s.id = t.statement_id
+    ${STATEMENTS_JOIN}
    LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
    LEFT JOIN transaction_splits ts ON ts.transaction_id = t.id AND ts.participant_id = $1
-    WHERE s.owner_id = $1
+    WHERE ${OWNER_SCOPE} = $1
      AND t.transaction_type IN ('debit', 'fee', 'interest', 'refund', 'credit')
      AND t.transaction_date >= $2
-      AND COALESCE(o.category_override, t.category) NOT IN ('transfers', 'investment')
+      AND ${EXCLUDE_NON_SPEND}
    GROUP BY 1
    HAVING SUM(${SPEND_EXPR}) > 0
    ORDER BY net_spend DESC
@@ -86,14 +87,14 @@ export async function GET(req: NextRequest) {
        TO_CHAR(DATE_TRUNC('month', t.transaction_date::date), 'YYYY-MM') as month,
        SUM(${SPEND_EXPR})::numeric(10,2) as total
      FROM transactions t
-      JOIN statements s ON s.id = t.statement_id
+      ${STATEMENTS_JOIN}
      LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
      LEFT JOIN transaction_splits ts ON ts.transaction_id = t.id AND ts.participant_id = $1
-      WHERE s.owner_id = $1
+      WHERE ${OWNER_SCOPE} = $1
        AND t.transaction_type IN ('debit', 'fee', 'interest', 'refund', 'credit')
        AND t.transaction_date >= $2
        AND COALESCE(o.merchant_normalized, t.merchant_normalized, t.merchant_name, t.description) = ANY($3)
-        AND COALESCE(o.category_override, t.category) NOT IN ('transfers', 'investment')
+        AND ${EXCLUDE_NON_SPEND}
      GROUP BY 1, 2
      ORDER BY 1, 2
    `, [user.id, fromDate, topMerchants]);
@@ -1,6 +1,12 @@
 import { NextRequest, NextResponse } from "next/server";
 import { getCurrentUser } from "@/lib/auth";
 import { queryRaw } from "@/lib/db";
+import {
+  OWNER_SCOPE,
+  STATEMENTS_JOIN,
+  EFFECTIVE_CATEGORY,
+  EXCLUDE_NON_SPEND,
+} from "@/lib/analytics-sql";

 export async function GET(req: NextRequest) {
  const user = await getCurrentUser(req);
@@ -25,7 +31,7 @@ export async function GET(req: NextRequest) {
  }>(
    `SELECT
      TO_CHAR(DATE_TRUNC('month', t.transaction_date::date), 'YYYY-MM') as month,
-      COALESCE(o.category_override, t.category) as category,
+      ${EFFECTIVE_CATEGORY} as category,
      SUM(
        CASE
          WHEN ts.share_percent IS NOT NULL THEN COALESCE(t.amount_aud, t.amount) * ts.share_percent / 100
@@ -37,10 +43,10 @@ export async function GET(req: NextRequest) {
    FROM transactions t
    LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
    LEFT JOIN transaction_splits ts ON ts.transaction_id = t.id AND ts.participant_id = $1
-    JOIN statements s ON s.id = t.statement_id
-    WHERE s.owner_id = $1
+    ${STATEMENTS_JOIN}
+    WHERE ${OWNER_SCOPE} = $1
      AND t.transaction_type IN ('debit', 'fee', 'interest')
-      AND COALESCE(o.category_override, t.category) NOT IN ('transfers', 'investment')
+      AND ${EXCLUDE_NON_SPEND}
      AND t.transaction_date >= $2
      AND t.transaction_date < $3
    GROUP BY 1, 2
@@ -60,10 +66,10 @@ export async function GET(req: NextRequest) {
      COUNT(*)::int as transaction_count
    FROM transactions t
    LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
-    JOIN statements s ON s.id = t.statement_id
-    WHERE s.owner_id = $1
+    ${STATEMENTS_JOIN}
+    WHERE ${OWNER_SCOPE} = $1
      AND t.transaction_type IN ('credit', 'payment')
-      AND COALESCE(o.category_override, t.category) = 'income'
+      AND ${EFFECTIVE_CATEGORY} = 'income'
      AND t.transaction_date >= $2
      AND t.transaction_date < $3
    GROUP BY 1
@@ -83,9 +89,9 @@ export async function GET(req: NextRequest) {
      COUNT(*)::int as transaction_count
    FROM transactions t
    LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
-    JOIN statements s ON s.id = t.statement_id
-    WHERE s.owner_id = $1
-      AND COALESCE(o.category_override, t.category) = 'investment'
+    ${STATEMENTS_JOIN}
+    WHERE ${OWNER_SCOPE} = $1
+      AND ${EFFECTIVE_CATEGORY} = 'investment'
      AND t.transaction_date >= $2
      AND t.transaction_date < $3
    GROUP BY 1
@@ -1,6 +1,7 @@
 import { NextRequest, NextResponse } from "next/server";
 import { getCurrentUser } from "@/lib/auth";
 import { queryRaw } from "@/lib/db";
+import { OWNER_SCOPE, STATEMENTS_JOIN, EXCLUDE_NON_SPEND, EFFECTIVE_CATEGORY } from "@/lib/analytics-sql";

 export async function GET(req: NextRequest) {
  const user = await getCurrentUser(req);
@@ -20,7 +21,7 @@ export async function GET(req: NextRequest) {
    `WITH merchant_txns AS (
      SELECT
        COALESCE(o.merchant_normalized, t.merchant_normalized, t.merchant_name) AS merchant,
-        COALESCE(o.category_override, t.category) AS category,
+        ${EFFECTIVE_CATEGORY} AS category,
        t.transaction_date,
        CASE
          WHEN ts.share_percent IS NOT NULL THEN COALESCE(t.amount_aud, t.amount) * ts.share_percent / 100
@@ -30,9 +31,10 @@ export async function GET(req: NextRequest) {
      FROM transactions t
      LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
      LEFT JOIN transaction_splits ts ON ts.transaction_id = t.id AND ts.participant_id = $1
-      JOIN statements s ON s.id = t.statement_id
-      WHERE s.owner_id = $1
+      ${STATEMENTS_JOIN}
+      WHERE ${OWNER_SCOPE} = $1
        AND t.transaction_type IN ('debit', 'fee')
+        AND ${EXCLUDE_NON_SPEND}
        AND COALESCE(o.merchant_normalized, t.merchant_normalized, t.merchant_name) IS NOT NULL
    ),
    merchant_with_lag AS (
@@ -48,7 +50,7 @@ export async function GET(req: NextRequest) {
      SELECT
        merchant,
        MODE() WITHIN GROUP (ORDER BY category) AS category,
-        COUNT(*) + 1 AS occurrences,
+        (COUNT(*) + 1)::int AS occurrences,
        AVG(my_amount)::numeric(12,2) AS avg_amount,
        MIN(transaction_date) AS first_seen,
        MAX(transaction_date) AS last_seen,
@@ -3,6 +3,12 @@
 import { useState, useMemo } from "react";
 import Link from "next/link";
 import { useStatements, useParticipants, useUpdateStatement } from "@/lib/hooks";
+import {
+  STATEMENT_TYPES,
+  STATEMENT_TYPE_LABELS,
+  asStatementType,
+  isLiability,
+} from "@/lib/statement-types";

 function formatDate(d: string | null) {
  if (!d) return "—";
@@ -40,7 +46,7 @@ export default function StatementsPage() {
  const updateStatement = useUpdateStatement();

  const [bankFilter, setBankFilter] = useState("");
-  const [typeFilter, setTypeFilter] = useState<"all" | "card" | "bank">("all");
+  const [typeFilter, setTypeFilter] = useState<"all" | (typeof STATEMENT_TYPES)[number]>("all");
  const [ownerFilter, setOwnerFilter] = useState("");
  const [yearFilter, setYearFilter] = useState("");

@@ -63,10 +69,8 @@ export default function StatementsPage() {
  const filtered = useMemo(() => {
    if (!statements) return [];
    return statements.filter((s) => {
-      const isCard = s.statement_type?.toLowerCase().includes("card") ?? false;
      if (bankFilter && s.bank_name !== bankFilter) return false;
-      if (typeFilter === "card" && !isCard) return false;
-      if (typeFilter === "bank" && isCard) return false;
+      if (typeFilter !== "all" && asStatementType(s.statement_type) !== typeFilter) return false;
      if (ownerFilter && String(s.owner_id) !== ownerFilter) return false;
      if (yearFilter && s.billing_end_date?.slice(0, 4) !== yearFilter) return false;
      return true;
@@ -98,8 +102,9 @@ export default function StatementsPage() {

          <select value={typeFilter} onChange={(e) => setTypeFilter(e.target.value as typeof typeFilter)} className={selectCls}>
            <option value="all">All types</option>
-            <option value="card">Credit card</option>
-            <option value="bank">Bank account</option>
+            {STATEMENT_TYPES.map((t) => (
+              <option key={t} value={t}>{STATEMENT_TYPE_LABELS[t]}</option>
+            ))}
          </select>

          {participants && participants.length > 1 && (
@@ -152,10 +157,14 @@ export default function StatementsPage() {
            </thead>
            <tbody>
              {filtered.map((s, idx) => {
-                const isCreditCard = s.statement_type?.toLowerCase().includes("card") ?? false;
-                const displayAmount = isCreditCard ? s.total_amount_due : s.closing_balance;
+                const stmtType = asStatementType(s.statement_type);
+                const owed = isLiability(s.statement_type);
+                // Cards headline the amount due; everything else (including loans,
+                // where the balance is what's still owed) headlines the balance.
+                const displayAmount =
+                  stmtType === "credit_card" ? s.total_amount_due : s.closing_balance;
                const amount = Number(displayAmount);
-                const amountColor = isCreditCard
+                const amountColor = owed
                  ? "text-red-400"
                  : amount >= 0
                  ? "text-green-400"
@@ -185,7 +194,7 @@ export default function StatementsPage() {
                      {formatPeriod(s.billing_start_date, s.billing_end_date)}
                    </td>
                    <td className="px-4 py-3 text-zinc-400 whitespace-nowrap">
-                      {isCreditCard ? formatDate(s.payment_due_date) : formatDate(s.billing_end_date)}
+                      {formatDate(s.payment_due_date ?? s.billing_end_date)}
                    </td>
                    <td className="px-4 py-3 text-zinc-500 text-xs">
                      {s.currency}