feat: CSV import and batch reconciliation UI
- Add reconciled_with_id column to transactions (links manual → statement tx) - CSV import wizard: 4-step modal (upload → map columns → review → done) - Handles any bank format via column mapping with localStorage presets - Single signed or separate debit/credit column modes - Editable preview table before committing - Auto-tags all imported rows with 'csv-import' - Batch reconcile page: shows all unreconciled manual transactions with potential statement matches (date ±3 days, amount ±1%) pre-fetched - Select matches across multiple rows, apply all at once - Copies overrides/tags/splits from manual → statement tx atomically - Manual tx marked reconciled (linked), hidden from main transactions view - Transactions with no matches shown separately - Import CSV button on transactions page - Reconcile nav item in sidebar
This commit is contained in:
@@ -0,0 +1,156 @@
|
||||
export type DateFormat = "DD/MM/YYYY" | "YYYY-MM-DD" | "MM/DD/YYYY" | "M/D/YYYY";
|
||||
|
||||
export interface ColumnMapping {
|
||||
dateCol: string;
|
||||
descriptionCol: string;
|
||||
amountMode: "single" | "debit_credit";
|
||||
amountCol?: string;
|
||||
debitCol?: string;
|
||||
creditCol?: string;
|
||||
merchantCol?: string;
|
||||
categoryCol?: string;
|
||||
}
|
||||
|
||||
export interface BankPreset {
|
||||
bankName: string;
|
||||
mapping: ColumnMapping;
|
||||
dateFormat: DateFormat;
|
||||
}
|
||||
|
||||
export interface ParsedTransaction {
|
||||
date: string;
|
||||
description: string;
|
||||
amount: number;
|
||||
transaction_type: string;
|
||||
merchant_name?: string;
|
||||
category?: string;
|
||||
}
|
||||
|
||||
export function parseCSVRows(text: string): string[][] {
|
||||
const rows: string[][] = [];
|
||||
let row: string[] = [];
|
||||
let field = "";
|
||||
let inQuotes = false;
|
||||
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const c = text[i];
|
||||
const next = text[i + 1];
|
||||
if (inQuotes) {
|
||||
if (c === '"' && next === '"') { field += '"'; i++; }
|
||||
else if (c === '"') { inQuotes = false; }
|
||||
else { field += c; }
|
||||
} else {
|
||||
if (c === '"') { inQuotes = true; }
|
||||
else if (c === ',') { row.push(field.trim()); field = ""; }
|
||||
else if (c === '\r' && next === '\n') {
|
||||
row.push(field.trim());
|
||||
if (row.some((f) => f !== "")) rows.push(row);
|
||||
row = []; field = ""; i++;
|
||||
} else if (c === '\n' || c === '\r') {
|
||||
row.push(field.trim());
|
||||
if (row.some((f) => f !== "")) rows.push(row);
|
||||
row = []; field = "";
|
||||
} else { field += c; }
|
||||
}
|
||||
}
|
||||
if (field || row.length > 0) {
|
||||
row.push(field.trim());
|
||||
if (row.some((f) => f !== "")) rows.push(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
export function parseDate(raw: string, format: DateFormat): string {
|
||||
const s = raw.trim();
|
||||
try {
|
||||
if (format === "DD/MM/YYYY") {
|
||||
const [d, m, y] = s.split("/");
|
||||
if (!d || !m || !y || y.length !== 4) return "";
|
||||
return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`;
|
||||
}
|
||||
if (format === "YYYY-MM-DD") {
|
||||
return /^\d{4}-\d{2}-\d{2}$/.test(s) ? s : "";
|
||||
}
|
||||
if (format === "MM/DD/YYYY" || format === "M/D/YYYY") {
|
||||
const [m, d, y] = s.split("/");
|
||||
if (!d || !m || !y || y.length !== 4) return "";
|
||||
return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`;
|
||||
}
|
||||
} catch { return ""; }
|
||||
return "";
|
||||
}
|
||||
|
||||
export function detectHasHeaders(rows: string[][], dateFormat: DateFormat): boolean {
|
||||
if (rows.length === 0) return false;
|
||||
return parseDate(rows[0][0] ?? "", dateFormat) === "";
|
||||
}
|
||||
|
||||
export function getColumnLabels(rows: string[][], hasHeaders: boolean): string[] {
|
||||
if (hasHeaders && rows.length > 0) {
|
||||
return rows[0].map((h, i) => h || `Column ${i + 1}`);
|
||||
}
|
||||
const maxCols = rows.reduce((m, r) => Math.max(m, r.length), 0);
|
||||
return Array.from({ length: maxCols }, (_, i) => `Column ${i + 1}`);
|
||||
}
|
||||
|
||||
export function getDataRows(rows: string[][], hasHeaders: boolean): string[][] {
|
||||
return hasHeaders ? rows.slice(1) : rows;
|
||||
}
|
||||
|
||||
export function applyMapping(
|
||||
dataRows: string[][],
|
||||
columnLabels: string[],
|
||||
mapping: ColumnMapping,
|
||||
dateFormat: DateFormat
|
||||
): ParsedTransaction[] {
|
||||
const idx = (name: string) => columnLabels.indexOf(name);
|
||||
const dateIdx = idx(mapping.dateCol);
|
||||
const descIdx = idx(mapping.descriptionCol);
|
||||
const merchantIdx = mapping.merchantCol ? idx(mapping.merchantCol) : -1;
|
||||
const categoryIdx = mapping.categoryCol ? idx(mapping.categoryCol) : -1;
|
||||
|
||||
const results: ParsedTransaction[] = [];
|
||||
for (const row of dataRows) {
|
||||
const date = parseDate(row[dateIdx] ?? "", dateFormat);
|
||||
const description = (row[descIdx] ?? "").trim();
|
||||
if (!date || !description) continue;
|
||||
|
||||
let amount = 0;
|
||||
let transaction_type = "debit";
|
||||
|
||||
if (mapping.amountMode === "single" && mapping.amountCol) {
|
||||
const raw = (row[idx(mapping.amountCol)] ?? "").replace(/[^\d.\-+]/g, "");
|
||||
const val = parseFloat(raw);
|
||||
if (isNaN(val) || val === 0) continue;
|
||||
amount = Math.abs(val);
|
||||
transaction_type = val < 0 ? "debit" : "credit";
|
||||
} else if (mapping.amountMode === "debit_credit") {
|
||||
const debitIdx = mapping.debitCol ? idx(mapping.debitCol) : -1;
|
||||
const creditIdx = mapping.creditCol ? idx(mapping.creditCol) : -1;
|
||||
const dVal = parseFloat((row[debitIdx] ?? "").replace(/[^\d.]/g, ""));
|
||||
const cVal = parseFloat((row[creditIdx] ?? "").replace(/[^\d.]/g, ""));
|
||||
if (!isNaN(dVal) && dVal > 0) { amount = dVal; transaction_type = "debit"; }
|
||||
else if (!isNaN(cVal) && cVal > 0) { amount = cVal; transaction_type = "credit"; }
|
||||
else continue;
|
||||
}
|
||||
|
||||
if (amount <= 0) continue;
|
||||
const tx: ParsedTransaction = { date, description, amount, transaction_type };
|
||||
if (merchantIdx >= 0 && row[merchantIdx]) tx.merchant_name = row[merchantIdx].trim();
|
||||
if (categoryIdx >= 0 && row[categoryIdx]) tx.category = row[categoryIdx].trim();
|
||||
results.push(tx);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
export function saveBankPreset(preset: BankPreset): void {
|
||||
try {
|
||||
const existing = loadBankPresets().filter((p) => p.bankName !== preset.bankName);
|
||||
localStorage.setItem("csv-presets", JSON.stringify([...existing, preset]));
|
||||
} catch { /* localStorage unavailable */ }
|
||||
}
|
||||
|
||||
export function loadBankPresets(): BankPreset[] {
|
||||
try { return JSON.parse(localStorage.getItem("csv-presets") || "[]"); }
|
||||
catch { return []; }
|
||||
}
|
||||
@@ -727,6 +727,66 @@ export interface MerchantTxnRow {
|
||||
statement_id: number;
|
||||
}
|
||||
|
||||
// --- CSV Import & Reconcile ---
|
||||
|
||||
export function useImportCSV() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: async (body: {
|
||||
bank_name: string;
|
||||
transactions: {
|
||||
date: string; description: string; amount: number; transaction_type: string;
|
||||
merchant_name?: string; foreign_currency_amount?: number; foreign_currency_code?: string; category?: string;
|
||||
}[];
|
||||
}) => {
|
||||
const res = await fetch("/api/import/csv", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
if (!res.ok) throw new Error((await res.json()).error || "Import failed");
|
||||
return res.json() as Promise<{ inserted: number }>;
|
||||
},
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: ["transactions"] });
|
||||
qc.invalidateQueries({ queryKey: ["tags"] });
|
||||
qc.invalidateQueries({ queryKey: ["reconcile-pending"] });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
import type { ManualTxWithMatches } from "./queries";
|
||||
|
||||
export function usePendingReconciliations() {
|
||||
return useQuery<ManualTxWithMatches[]>({
|
||||
queryKey: ["reconcile-pending"],
|
||||
queryFn: async () => {
|
||||
const res = await fetch("/api/reconcile/pending");
|
||||
if (!res.ok) throw new Error("Failed to fetch");
|
||||
return res.json();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useReconcile() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: async (matches: { manual_id: number; statement_tx_id: number }[]) => {
|
||||
const res = await fetch("/api/transactions/reconcile", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ matches }),
|
||||
});
|
||||
if (!res.ok) throw new Error((await res.json()).error || "Reconcile failed");
|
||||
return res.json() as Promise<{ reconciled: number }>;
|
||||
},
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: ["transactions"] });
|
||||
qc.invalidateQueries({ queryKey: ["reconcile-pending"] });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useMerchantTransactions(merchant: string | null) {
|
||||
return useQuery<{ transactions: MerchantTxnRow[] }>({
|
||||
queryKey: ["analytics", "merchant-txns", merchant],
|
||||
|
||||
+180
-1
@@ -85,7 +85,10 @@ interface TransactionFilters {
|
||||
}
|
||||
|
||||
export async function getTransactions(ownerId: number, filters: TransactionFilters) {
|
||||
const conditions: string[] = [`(COALESCE(t.owner_id, s.owner_id) = $1 OR EXISTS (SELECT 1 FROM transaction_splits ts_me WHERE ts_me.transaction_id = t.id AND ts_me.participant_id = $1))`];
|
||||
const conditions: string[] = [
|
||||
`(COALESCE(t.owner_id, s.owner_id) = $1 OR EXISTS (SELECT 1 FROM transaction_splits ts_me WHERE ts_me.transaction_id = t.id AND ts_me.participant_id = $1))`,
|
||||
`NOT (t.statement_id IS NULL AND t.reconciled_with_id IS NOT NULL)`,
|
||||
];
|
||||
const params: unknown[] = [ownerId];
|
||||
let paramIdx = 2;
|
||||
|
||||
@@ -344,6 +347,182 @@ export interface SharedTransactionRow extends TransactionRow {
|
||||
splits: { participant_id: number; name: string; share_percent: number; settled: boolean }[];
|
||||
}
|
||||
|
||||
export async function ensureTag(name: string, color: string): Promise<number> {
|
||||
const rows = await queryRaw<{ id: number }>(
|
||||
`INSERT INTO tags (name, color) VALUES ($1, $2)
|
||||
ON CONFLICT (name) DO UPDATE SET name = EXCLUDED.name
|
||||
RETURNING id`,
|
||||
[name, color]
|
||||
);
|
||||
return rows[0].id;
|
||||
}
|
||||
|
||||
export async function batchInsertCSVTransactions(
|
||||
ownerId: number,
|
||||
rows: {
|
||||
date: string;
|
||||
description: string;
|
||||
amount: number;
|
||||
transaction_type: string;
|
||||
merchant_name?: string;
|
||||
foreign_currency_amount?: number;
|
||||
foreign_currency_code?: string;
|
||||
category?: string;
|
||||
}[],
|
||||
tagId: number
|
||||
): Promise<number> {
|
||||
if (rows.length === 0) return 0;
|
||||
|
||||
const baseRows = await queryRaw<{ base: number }>(
|
||||
`SELECT COALESCE(MAX(row_index), -1) as base FROM transactions WHERE owner_id = $1 AND statement_id IS NULL`,
|
||||
[ownerId]
|
||||
);
|
||||
const base = Number(baseRows[0].base);
|
||||
|
||||
const valueClauses: string[] = [];
|
||||
const params: unknown[] = [ownerId];
|
||||
let p = 2;
|
||||
rows.forEach((r, i) => {
|
||||
valueClauses.push(`(NULL, $1, $${p++}, $${p++}, $${p++}, $${p++}, $${p++}, $${p++}, $${p++}, ${base + 1 + i})`);
|
||||
params.push(r.date, r.description, r.amount, r.transaction_type, r.merchant_name ?? null, r.foreign_currency_amount ?? null, r.foreign_currency_code ?? null);
|
||||
});
|
||||
|
||||
const txIds = await queryRaw<{ id: number }>(
|
||||
`INSERT INTO transactions (statement_id, owner_id, transaction_date, description, amount, transaction_type, merchant_name, foreign_currency_amount, foreign_currency_code, row_index)
|
||||
VALUES ${valueClauses.join(", ")}
|
||||
RETURNING id`,
|
||||
params
|
||||
);
|
||||
|
||||
if (txIds.length > 0) {
|
||||
const tagValueClauses = txIds.map((_, i) => `($${i * 2 + 1}, $${i * 2 + 2})`);
|
||||
const tagParams: unknown[] = txIds.flatMap((r) => [r.id, tagId]);
|
||||
await queryRaw(
|
||||
`INSERT INTO transaction_tags (transaction_id, tag_id) VALUES ${tagValueClauses.join(", ")} ON CONFLICT DO NOTHING`,
|
||||
tagParams
|
||||
);
|
||||
}
|
||||
|
||||
return txIds.length;
|
||||
}
|
||||
|
||||
export interface PotentialMatch {
|
||||
id: number;
|
||||
transaction_date: string;
|
||||
description: string;
|
||||
amount: number;
|
||||
transaction_type: string;
|
||||
effective_merchant: string;
|
||||
effective_category: string;
|
||||
bank_name: string;
|
||||
billing_end_date: string | null;
|
||||
}
|
||||
|
||||
export interface ManualTxWithMatches extends TransactionRow {
|
||||
matches: PotentialMatch[];
|
||||
}
|
||||
|
||||
export async function getPendingReconciliations(ownerId: number): Promise<ManualTxWithMatches[]> {
|
||||
// Fetch all unreconciled manual transactions
|
||||
const raw = await queryRaw<TransactionRow & { tags: string | TagRow[]; splits: string | TransactionRow["splits"] }>(
|
||||
`SELECT t.*,
|
||||
o.category_override, o.merchant_normalized as merchant_override, o.notes, o.my_share_percent,
|
||||
COALESCE(o.category_override, t.category) as effective_category,
|
||||
COALESCE(o.merchant_normalized, t.merchant_normalized, t.merchant_name) as effective_merchant,
|
||||
'Manual' as bank_name,
|
||||
t.owner_id,
|
||||
p.name as owner_name,
|
||||
txn_tags.tags,
|
||||
txn_splits.splits
|
||||
FROM transactions t
|
||||
LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
|
||||
LEFT JOIN participants p ON p.id = t.owner_id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT COALESCE(json_agg(json_build_object('id', tg.id, 'name', tg.name, 'color', tg.color) ORDER BY tg.name), '[]'::json) as tags
|
||||
FROM transaction_tags tt JOIN tags tg ON tg.id = tt.tag_id
|
||||
WHERE tt.transaction_id = t.id
|
||||
) txn_tags ON true
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT COALESCE(json_agg(json_build_object('participant_id', ts.participant_id, 'name', sp.name, 'share_percent', ts.share_percent, 'settled', ts.settled) ORDER BY sp.name), '[]'::json) as splits
|
||||
FROM transaction_splits ts JOIN participants sp ON sp.id = ts.participant_id
|
||||
WHERE ts.transaction_id = t.id
|
||||
) txn_splits ON true
|
||||
WHERE t.statement_id IS NULL AND t.owner_id = $1 AND t.reconciled_with_id IS NULL
|
||||
ORDER BY t.transaction_date DESC, t.row_index ASC`,
|
||||
[ownerId]
|
||||
);
|
||||
|
||||
const manualTxs = raw.map((r) => ({
|
||||
...r,
|
||||
tags: typeof r.tags === "string" ? JSON.parse(r.tags) : (r.tags ?? []),
|
||||
splits: typeof r.splits === "string" ? JSON.parse(r.splits) : (r.splits ?? []),
|
||||
})) as TransactionRow[];
|
||||
|
||||
if (manualTxs.length === 0) return [];
|
||||
|
||||
// Fetch all potential matches in one query using window function
|
||||
const matchRows = await queryRaw<PotentialMatch & { manual_id: number; rn: number }>(
|
||||
`SELECT manual_id, id, transaction_date, description, amount, transaction_type,
|
||||
effective_merchant, effective_category, bank_name, billing_end_date
|
||||
FROM (
|
||||
SELECT
|
||||
m.id AS manual_id,
|
||||
t.id,
|
||||
t.transaction_date,
|
||||
t.description,
|
||||
t.amount,
|
||||
t.transaction_type,
|
||||
COALESCE(o.merchant_normalized, t.merchant_normalized, t.merchant_name, '') AS effective_merchant,
|
||||
COALESCE(o.category_override, t.category, '') AS effective_category,
|
||||
s.bank_name,
|
||||
s.billing_end_date,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY m.id
|
||||
ORDER BY ABS(t.amount - m.amount), ABS(t.transaction_date - m.transaction_date)
|
||||
) AS rn
|
||||
FROM transactions m
|
||||
JOIN transactions t ON t.statement_id IS NOT NULL
|
||||
AND t.transaction_date BETWEEN m.transaction_date - 3 AND m.transaction_date + 3
|
||||
AND t.amount BETWEEN m.amount * 0.99 AND m.amount * 1.01
|
||||
JOIN statements s ON s.id = t.statement_id
|
||||
LEFT JOIN transaction_overrides o ON o.transaction_id = t.id
|
||||
WHERE m.statement_id IS NULL
|
||||
AND m.owner_id = $1
|
||||
AND m.reconciled_with_id IS NULL
|
||||
AND COALESCE(t.owner_id, s.owner_id) = $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM transactions mt WHERE mt.reconciled_with_id = t.id
|
||||
)
|
||||
) sq
|
||||
WHERE rn <= 5
|
||||
ORDER BY manual_id, rn`,
|
||||
[ownerId]
|
||||
);
|
||||
|
||||
// Group matches by manual_id
|
||||
const matchesByManualId = new Map<number, PotentialMatch[]>();
|
||||
for (const row of matchRows) {
|
||||
const list = matchesByManualId.get(row.manual_id) ?? [];
|
||||
list.push({
|
||||
id: row.id,
|
||||
transaction_date: row.transaction_date,
|
||||
description: row.description,
|
||||
amount: row.amount,
|
||||
transaction_type: row.transaction_type,
|
||||
effective_merchant: row.effective_merchant,
|
||||
effective_category: row.effective_category,
|
||||
bank_name: row.bank_name,
|
||||
billing_end_date: row.billing_end_date,
|
||||
});
|
||||
matchesByManualId.set(row.manual_id, list);
|
||||
}
|
||||
|
||||
return manualTxs.map((tx) => ({
|
||||
...tx,
|
||||
matches: matchesByManualId.get(tx.id) ?? [],
|
||||
}));
|
||||
}
|
||||
|
||||
export async function getTags() {
|
||||
return queryRaw<TagRow & { transaction_count: number }>(`
|
||||
SELECT tg.id, tg.name, tg.color,
|
||||
|
||||
Reference in New Issue
Block a user