feat(statements): flag billing periods that overlap another statement
ci / lint-test (push) Successful in 46s

An account cannot be billed twice for the same day, so an overlap means those
transactions are in the ledger twice. ANZ statements 107 and 143 overlap by 118
days and put roughly $42,000 of duplicate rows in; nothing anywhere said so.

Two details decide whether this catches the real case:

  - Account numbers compare with non-digits stripped. The duplicate got in
    because the existing key compared raw text and ANZ wrote the same account
    as 408556264 on one statement and 4085-56264 on the other.
  - The range is half-open. These statements are issued back-to-back with one
    period ending the day the next starts, so inclusive bounds flagged 5 pairs
    of which 3 were consecutive and fine. Half-open leaves exactly the 2 real
    ones.

NULL bounds are excluded rather than handed to daterange, where NULL means
unbounded and an undated statement would overlap all of history.

Detection only. It does not refuse the import or touch the duplicate rows --
cleaning those is separate, and must supersede rather than delete because every
child of transactions is ON DELETE CASCADE and the curation sits on the
duplicate side.

Both subtleties have a test, and both fail if you undo them.
This commit is contained in:
2026-07-28 11:46:45 +10:00
parent 7a1acc32a9
commit d5589b2980
3 changed files with 134 additions and 2 deletions
+71 -1
View File
@@ -8,7 +8,7 @@ mockDbWithPool(pool);
// Dynamic import AFTER the mock ensures getTransactions / getParticipantBalances // Dynamic import AFTER the mock ensures getTransactions / getParticipantBalances
// use the test pool rather than Prisma's singleton. // use the test pool rather than Prisma's singleton.
const { getTransactions, getParticipantBalances, getTripAnalytics, getTripById } = await import("@/lib/queries"); const { getTransactions, getParticipantBalances, getTripAnalytics, getTripById, getStatements } = await import("@/lib/queries");
beforeEach(async () => { beforeEach(async () => {
await resetDB(pool); await resetDB(pool);
@@ -538,3 +538,73 @@ describe("getTripAnalytics — refunds reduce trip cost", () => {
expect(Number(bob!.owed)).toBeCloseTo(100); expect(Number(bob!.owed)).toBeCloseTo(100);
}); });
}); });
// An account cannot be billed twice for the same day. The boundary handling is
// the whole difficulty: these statements are issued back-to-back with one
// period ending the day the next begins, so naive inclusive ranges flag every
// consecutive pair.
describe("getStatements — overlapping billing periods", () => {
async function addStatement(
ownerId: number, account: string, start: string | null, end: string | null
): Promise<number> {
const r = await pool.query(
`INSERT INTO statements (filename, bank_name, account_number, owner_id,
billing_start_date, billing_end_date)
VALUES ($1, 'ANZ', $2, $3, $4, $5) RETURNING id`,
[`stmt-${account}-${start}.pdf`, account, ownerId, start, end]
);
return r.rows[0].id as number;
}
it("does not flag statements that merely touch at a boundary", async () => {
const { ownerId } = await seedParticipants(pool);
await addStatement(ownerId, "4085-56264", "2025-05-16", "2025-11-14");
await addStatement(ownerId, "4085-56264", "2025-11-14", "2026-05-15");
const rows = await getStatements(ownerId);
expect(rows.every((r) => r.overlaps.length === 0)).toBe(true);
});
it("flags a genuine overlap on both statements, with the day count", async () => {
const { ownerId } = await seedParticipants(pool);
const a = await addStatement(ownerId, "4085-56264", "2025-11-12", "2026-03-12");
const b = await addStatement(ownerId, "4085-56264", "2025-11-14", "2026-05-15");
const rows = await getStatements(ownerId);
const rowA = rows.find((r) => r.id === a)!;
const rowB = rows.find((r) => r.id === b)!;
expect(rowA.overlaps).toEqual([{ id: b, days: 118 }]);
expect(rowB.overlaps).toEqual([{ id: a, days: 118 }]);
});
// The real duplicate got in because the existing key compared raw text and
// ANZ wrote the same account both ways.
it("matches the same account written with and without punctuation", async () => {
const { ownerId } = await seedParticipants(pool);
const a = await addStatement(ownerId, "408556264", "2025-11-12", "2026-03-12");
const b = await addStatement(ownerId, "4085-56264", "2025-11-14", "2026-05-15");
const rows = await getStatements(ownerId);
expect(rows.find((r) => r.id === a)!.overlaps).toEqual([{ id: b, days: 118 }]);
});
it("ignores a different account billing the same days", async () => {
const { ownerId } = await seedParticipants(pool);
await addStatement(ownerId, "4085-56264", "2025-11-12", "2026-03-12");
await addStatement(ownerId, "9999-11111", "2025-11-12", "2026-03-12");
const rows = await getStatements(ownerId);
expect(rows.every((r) => r.overlaps.length === 0)).toBe(true);
});
// NULL is unbounded to daterange, which would make an undated statement
// overlap the entire history.
it("does not treat an undated statement as overlapping everything", async () => {
const { ownerId } = await seedParticipants(pool);
await addStatement(ownerId, "4085-56264", "2025-11-12", "2026-03-12");
await addStatement(ownerId, "4085-56264", null, null);
const rows = await getStatements(ownerId);
expect(rows.every((r) => r.overlaps.length === 0)).toBe(true);
});
});
+15
View File
@@ -206,6 +206,21 @@ export default function StatementsPage() {
</td> </td>
<td className="px-4 py-3 text-zinc-400 whitespace-nowrap"> <td className="px-4 py-3 text-zinc-400 whitespace-nowrap">
{formatPeriod(s.billing_start_date, s.billing_end_date)} {formatPeriod(s.billing_start_date, s.billing_end_date)}
{/* An account cannot be billed twice for the same day, so
an overlap means these transactions are in the ledger
twice. Red rather than amber: the balance warning above
means a statement doesn't add up, this means the data is
double-counted everywhere it is summed. */}
{s.overlaps?.length > 0 && (
<div
className="text-[10px] text-red-400 mt-0.5"
title={`Billing period overlaps statement ${s.overlaps
.map((o) => `#${o.id} by ${o.days} day${o.days === 1 ? "" : "s"}`)
.join(", ")}. The overlapping transactions are likely imported twice.`}
>
overlaps #{s.overlaps.map((o) => o.id).join(", #")}
</div>
)}
</td> </td>
<td className="px-4 py-3 text-zinc-400 whitespace-nowrap"> <td className="px-4 py-3 text-zinc-400 whitespace-nowrap">
{formatDate(s.payment_due_date ?? s.billing_end_date)} {formatDate(s.payment_due_date ?? s.billing_end_date)}
+48 -1
View File
@@ -105,6 +105,12 @@ export interface StatementRow {
// opening/closing balance to check against. // opening/closing balance to check against.
expected_closing: number | null; expected_closing: number | null;
balance_diff: number | null; balance_diff: number | null;
/**
* Other statements for this account billing the same days — see
* STATEMENT_OVERLAPS. Non-empty means some of these transactions are almost
* certainly imported twice. Empty array, never null.
*/
overlaps: { id: number; days: number }[];
} }
interface TransactionFilters { interface TransactionFilters {
@@ -361,15 +367,56 @@ export const BALANCE_DELTA = `SUM(CASE
ELSE CASE WHEN t.transaction_type IN ('debit', 'fee', 'interest') THEN -t.amount ELSE t.amount END ELSE CASE WHEN t.transaction_type IN ('debit', 'fee', 'interest') THEN -t.amount ELSE t.amount END
END)`; END)`;
/**
* Other statements for the same account whose billing period overlaps this one.
*
* An account cannot be billed twice for the same day, so an overlap means the
* same transactions were imported twice. This is not hypothetical: ANZ
* statements 107 and 143 overlap by 118 days and put ~$42,000 of duplicate rows
* in the ledger, which silently inflated spend and dragged the CSV split match
* rate down to 46.5%.
*
* Two details are what make it actually catch that case:
*
* - The account number is compared with non-digits stripped. The duplicate got
* in precisely because the existing duplicate key compared raw text, and ANZ
* wrote the same account as `408556264` on one statement and `4085-56264` on
* the other.
* - The range is half-open `[)`. These statements are issued back-to-back with
* one period's end date equal to the next one's start, so inclusive bounds
* flag every consecutive pair — 5 hits of which 3 were false. Half-open
* leaves exactly the 2 real ones.
*
* NULL bounds are excluded rather than passed to `daterange`, where NULL means
* unbounded and would make an undated statement overlap the entire history.
*/
const STATEMENT_OVERLAPS = `
SELECT COALESCE(json_agg(json_build_object(
'id', o.id,
'days', (LEAST(s.billing_end_date, o.billing_end_date)
- GREATEST(s.billing_start_date, o.billing_start_date))
) ORDER BY o.id), '[]'::json) AS overlaps
FROM statements o
WHERE o.id <> s.id
AND o.owner_id = s.owner_id
AND regexp_replace(o.account_number, '\\D', '', 'g')
= regexp_replace(s.account_number, '\\D', '', 'g')
AND o.billing_start_date IS NOT NULL AND o.billing_end_date IS NOT NULL
AND s.billing_start_date IS NOT NULL AND s.billing_end_date IS NOT NULL
AND daterange(s.billing_start_date, s.billing_end_date, '[)')
&& daterange(o.billing_start_date, o.billing_end_date, '[)')`;
export async function getStatements(ownerId: number) { export async function getStatements(ownerId: number) {
const sql = ` const sql = `
SELECT s.*, SELECT s.*,
(SELECT COUNT(*)::int FROM transactions t WHERE t.statement_id = s.id) as transaction_count, (SELECT COUNT(*)::int FROM transactions t WHERE t.statement_id = s.id) as transaction_count,
p.name as owner_name, p.name as owner_name,
recon.expected_closing, recon.expected_closing,
recon.balance_diff recon.balance_diff,
ov.overlaps
FROM statements s FROM statements s
LEFT JOIN participants p ON p.id = s.owner_id LEFT JOIN participants p ON p.id = s.owner_id
LEFT JOIN LATERAL (${STATEMENT_OVERLAPS}) ov ON true
LEFT JOIN LATERAL ( LEFT JOIN LATERAL (
SELECT SELECT
(s.opening_balance + ${BALANCE_DELTA})::numeric(12,2) as expected_closing, (s.opening_balance + ${BALANCE_DELTA})::numeric(12,2) as expected_closing,