feat(statements): flag billing periods that overlap another statement
ci / lint-test (push) Successful in 46s
ci / lint-test (push) Successful in 46s
An account cannot be billed twice for the same day, so an overlap means those
transactions are in the ledger twice. ANZ statements 107 and 143 overlap by 118
days and put roughly $42,000 of duplicate rows in; nothing anywhere said so.
Two details decide whether this catches the real case:
- Account numbers compare with non-digits stripped. The duplicate got in
because the existing key compared raw text and ANZ wrote the same account
as 408556264 on one statement and 4085-56264 on the other.
- The range is half-open. These statements are issued back-to-back with one
period ending the day the next starts, so inclusive bounds flagged 5 pairs
of which 3 were consecutive and fine. Half-open leaves exactly the 2 real
ones.
NULL bounds are excluded rather than handed to daterange, where NULL means
unbounded and an undated statement would overlap all of history.
Detection only. It does not refuse the import or touch the duplicate rows --
cleaning those is separate, and must supersede rather than delete because every
child of transactions is ON DELETE CASCADE and the curation sits on the
duplicate side.
Both subtleties have a test, and both fail if you undo them.
This commit is contained in:
@@ -8,7 +8,7 @@ mockDbWithPool(pool);
|
||||
|
||||
// Dynamic import AFTER the mock ensures getTransactions / getParticipantBalances
|
||||
// use the test pool rather than Prisma's singleton.
|
||||
const { getTransactions, getParticipantBalances, getTripAnalytics, getTripById } = await import("@/lib/queries");
|
||||
const { getTransactions, getParticipantBalances, getTripAnalytics, getTripById, getStatements } = await import("@/lib/queries");
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetDB(pool);
|
||||
@@ -538,3 +538,73 @@ describe("getTripAnalytics — refunds reduce trip cost", () => {
|
||||
expect(Number(bob!.owed)).toBeCloseTo(100);
|
||||
});
|
||||
});
|
||||
|
||||
// An account cannot be billed twice for the same day. The boundary handling is
|
||||
// the whole difficulty: these statements are issued back-to-back with one
|
||||
// period ending the day the next begins, so naive inclusive ranges flag every
|
||||
// consecutive pair.
|
||||
describe("getStatements — overlapping billing periods", () => {
|
||||
async function addStatement(
|
||||
ownerId: number, account: string, start: string | null, end: string | null
|
||||
): Promise<number> {
|
||||
const r = await pool.query(
|
||||
`INSERT INTO statements (filename, bank_name, account_number, owner_id,
|
||||
billing_start_date, billing_end_date)
|
||||
VALUES ($1, 'ANZ', $2, $3, $4, $5) RETURNING id`,
|
||||
[`stmt-${account}-${start}.pdf`, account, ownerId, start, end]
|
||||
);
|
||||
return r.rows[0].id as number;
|
||||
}
|
||||
|
||||
it("does not flag statements that merely touch at a boundary", async () => {
|
||||
const { ownerId } = await seedParticipants(pool);
|
||||
await addStatement(ownerId, "4085-56264", "2025-05-16", "2025-11-14");
|
||||
await addStatement(ownerId, "4085-56264", "2025-11-14", "2026-05-15");
|
||||
|
||||
const rows = await getStatements(ownerId);
|
||||
expect(rows.every((r) => r.overlaps.length === 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("flags a genuine overlap on both statements, with the day count", async () => {
|
||||
const { ownerId } = await seedParticipants(pool);
|
||||
const a = await addStatement(ownerId, "4085-56264", "2025-11-12", "2026-03-12");
|
||||
const b = await addStatement(ownerId, "4085-56264", "2025-11-14", "2026-05-15");
|
||||
|
||||
const rows = await getStatements(ownerId);
|
||||
const rowA = rows.find((r) => r.id === a)!;
|
||||
const rowB = rows.find((r) => r.id === b)!;
|
||||
expect(rowA.overlaps).toEqual([{ id: b, days: 118 }]);
|
||||
expect(rowB.overlaps).toEqual([{ id: a, days: 118 }]);
|
||||
});
|
||||
|
||||
// The real duplicate got in because the existing key compared raw text and
|
||||
// ANZ wrote the same account both ways.
|
||||
it("matches the same account written with and without punctuation", async () => {
|
||||
const { ownerId } = await seedParticipants(pool);
|
||||
const a = await addStatement(ownerId, "408556264", "2025-11-12", "2026-03-12");
|
||||
const b = await addStatement(ownerId, "4085-56264", "2025-11-14", "2026-05-15");
|
||||
|
||||
const rows = await getStatements(ownerId);
|
||||
expect(rows.find((r) => r.id === a)!.overlaps).toEqual([{ id: b, days: 118 }]);
|
||||
});
|
||||
|
||||
it("ignores a different account billing the same days", async () => {
|
||||
const { ownerId } = await seedParticipants(pool);
|
||||
await addStatement(ownerId, "4085-56264", "2025-11-12", "2026-03-12");
|
||||
await addStatement(ownerId, "9999-11111", "2025-11-12", "2026-03-12");
|
||||
|
||||
const rows = await getStatements(ownerId);
|
||||
expect(rows.every((r) => r.overlaps.length === 0)).toBe(true);
|
||||
});
|
||||
|
||||
// NULL is unbounded to daterange, which would make an undated statement
|
||||
// overlap the entire history.
|
||||
it("does not treat an undated statement as overlapping everything", async () => {
|
||||
const { ownerId } = await seedParticipants(pool);
|
||||
await addStatement(ownerId, "4085-56264", "2025-11-12", "2026-03-12");
|
||||
await addStatement(ownerId, "4085-56264", null, null);
|
||||
|
||||
const rows = await getStatements(ownerId);
|
||||
expect(rows.every((r) => r.overlaps.length === 0)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -206,6 +206,21 @@ export default function StatementsPage() {
|
||||
</td>
|
||||
<td className="px-4 py-3 text-zinc-400 whitespace-nowrap">
|
||||
{formatPeriod(s.billing_start_date, s.billing_end_date)}
|
||||
{/* An account cannot be billed twice for the same day, so
|
||||
an overlap means these transactions are in the ledger
|
||||
twice. Red rather than amber: the balance warning above
|
||||
means a statement doesn't add up, this means the data is
|
||||
double-counted everywhere it is summed. */}
|
||||
{s.overlaps?.length > 0 && (
|
||||
<div
|
||||
className="text-[10px] text-red-400 mt-0.5"
|
||||
title={`Billing period overlaps statement ${s.overlaps
|
||||
.map((o) => `#${o.id} by ${o.days} day${o.days === 1 ? "" : "s"}`)
|
||||
.join(", ")}. The overlapping transactions are likely imported twice.`}
|
||||
>
|
||||
⚠ overlaps #{s.overlaps.map((o) => o.id).join(", #")}
|
||||
</div>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-4 py-3 text-zinc-400 whitespace-nowrap">
|
||||
{formatDate(s.payment_due_date ?? s.billing_end_date)}
|
||||
|
||||
+48
-1
@@ -105,6 +105,12 @@ export interface StatementRow {
|
||||
// opening/closing balance to check against.
|
||||
expected_closing: number | null;
|
||||
balance_diff: number | null;
|
||||
/**
|
||||
* Other statements for this account billing the same days — see
|
||||
* STATEMENT_OVERLAPS. Non-empty means some of these transactions are almost
|
||||
* certainly imported twice. Empty array, never null.
|
||||
*/
|
||||
overlaps: { id: number; days: number }[];
|
||||
}
|
||||
|
||||
interface TransactionFilters {
|
||||
@@ -361,15 +367,56 @@ export const BALANCE_DELTA = `SUM(CASE
|
||||
ELSE CASE WHEN t.transaction_type IN ('debit', 'fee', 'interest') THEN -t.amount ELSE t.amount END
|
||||
END)`;
|
||||
|
||||
/**
|
||||
* Other statements for the same account whose billing period overlaps this one.
|
||||
*
|
||||
* An account cannot be billed twice for the same day, so an overlap means the
|
||||
* same transactions were imported twice. This is not hypothetical: ANZ
|
||||
* statements 107 and 143 overlap by 118 days and put ~$42,000 of duplicate rows
|
||||
* in the ledger, which silently inflated spend and dragged the CSV split match
|
||||
* rate down to 46.5%.
|
||||
*
|
||||
* Two details are what make it actually catch that case:
|
||||
*
|
||||
* - The account number is compared with non-digits stripped. The duplicate got
|
||||
* in precisely because the existing duplicate key compared raw text, and ANZ
|
||||
* wrote the same account as `408556264` on one statement and `4085-56264` on
|
||||
* the other.
|
||||
* - The range is half-open `[)`. These statements are issued back-to-back with
|
||||
* one period's end date equal to the next one's start, so inclusive bounds
|
||||
* flag every consecutive pair — 5 hits of which 3 were false. Half-open
|
||||
* leaves exactly the 2 real ones.
|
||||
*
|
||||
* NULL bounds are excluded rather than passed to `daterange`, where NULL means
|
||||
* unbounded and would make an undated statement overlap the entire history.
|
||||
*/
|
||||
const STATEMENT_OVERLAPS = `
|
||||
SELECT COALESCE(json_agg(json_build_object(
|
||||
'id', o.id,
|
||||
'days', (LEAST(s.billing_end_date, o.billing_end_date)
|
||||
- GREATEST(s.billing_start_date, o.billing_start_date))
|
||||
) ORDER BY o.id), '[]'::json) AS overlaps
|
||||
FROM statements o
|
||||
WHERE o.id <> s.id
|
||||
AND o.owner_id = s.owner_id
|
||||
AND regexp_replace(o.account_number, '\\D', '', 'g')
|
||||
= regexp_replace(s.account_number, '\\D', '', 'g')
|
||||
AND o.billing_start_date IS NOT NULL AND o.billing_end_date IS NOT NULL
|
||||
AND s.billing_start_date IS NOT NULL AND s.billing_end_date IS NOT NULL
|
||||
AND daterange(s.billing_start_date, s.billing_end_date, '[)')
|
||||
&& daterange(o.billing_start_date, o.billing_end_date, '[)')`;
|
||||
|
||||
export async function getStatements(ownerId: number) {
|
||||
const sql = `
|
||||
SELECT s.*,
|
||||
(SELECT COUNT(*)::int FROM transactions t WHERE t.statement_id = s.id) as transaction_count,
|
||||
p.name as owner_name,
|
||||
recon.expected_closing,
|
||||
recon.balance_diff
|
||||
recon.balance_diff,
|
||||
ov.overlaps
|
||||
FROM statements s
|
||||
LEFT JOIN participants p ON p.id = s.owner_id
|
||||
LEFT JOIN LATERAL (${STATEMENT_OVERLAPS}) ov ON true
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT
|
||||
(s.opening_balance + ${BALANCE_DELTA})::numeric(12,2) as expected_closing,
|
||||
|
||||
Reference in New Issue
Block a user