From 5e355c7e859dc9eda9d703a948656c0da8432dee Mon Sep 17 00:00:00 2001 From: siddharthd Date: Sat, 22 Aug 2026 16:03:06 +1000 Subject: [PATCH] currency: normalise foreign_currency_code to ISO-4217 (board #254) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exact-code equality is the whole basis of FX reconciliation — order_transaction_bridge matches on foreign_currency_code = %(ccy)s — so a row labelled 'BAHT' could never match a THB order and a row labelled 'null' could never match anything. 226 of 1,632 coded rows (14%) were unmatchable for this reason alone, silently degrading a shipped job. 187 rows carried the literal 4-character STRING 'null' rather than SQL NULL, 42 of those also carrying foreign_currency_amount = -1.00 exactly. Root cause is a truthy guard in the CC processors' Parse node — 't.foreign_currency_code || null' passes the string "null" straight through, because a non-empty string is truthy. Not historical: 155 rows in 2026-03 but also 8 in May, 3 in July and 21 in August. Fixed at source in the same change (smarthome cc-statement-processor-paperless.json). Mappings were verified against IMPLIED RATE, not trusted from the label, because a row whose rate contradicts its currency is a broken ledger row rather than a mislabelled one — the trap order_transaction_bridge's rate guard exists for. All six agreed: BAHT 0.0433 vs THB 0.0470, INDIAN RUPEE 0.0183 vs INR 0.0180, U.A.E. DIRHAM 0.4287 vs AED 0.4184, US DOLLAR 1.4054 vs USD 1.5513, NZ DOLLAR 0.8243 vs NZD 0.9311, EURO 1.6342 vs EUR 1.6964. foreign_currency_amount is untouched by the relabels — only the code was ever wrong, so this is a relabel, never a recompute. Applied row counts matched the pre-flight exactly: 42, 187, 13, 10, 6, 4, 4, 2. Postcondition asserted inside the transaction: 0 non-ISO codes remain. Verified after: 0 'null' strings, 0 -1.00 sentinels, 1445 coded rows (1632 - 187). --- .../migration.sql | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 prisma/migrations/0036_normalize_foreign_currency_code/migration.sql diff --git a/prisma/migrations/0036_normalize_foreign_currency_code/migration.sql b/prisma/migrations/0036_normalize_foreign_currency_code/migration.sql new file mode 100644 index 0000000..affd250 --- /dev/null +++ b/prisma/migrations/0036_normalize_foreign_currency_code/migration.sql @@ -0,0 +1,73 @@ +-- Normalise foreign_currency_code to ISO-4217, and clear two sentinel values. +-- +-- WHY THIS MATTERS: exact-code equality is the whole basis of FX reconciliation. +-- ingestion-engine's order_transaction_bridge matches foreign orders with +-- AND t.foreign_currency_code = %(ccy)s +-- AND t.foreign_currency_amount = %(total)s +-- so a row labelled 'BAHT' can never match a THB order, and a row labelled +-- 'null' can never match anything. 226 of 1,632 coded rows (14%) were +-- unmatchable for this reason alone, which silently degrades a shipped job. +-- +-- THE SENTINELS. 187 rows carry the literal 4-character STRING 'null' rather +-- than SQL NULL, and 42 of those also carry foreign_currency_amount = -1.00 +-- exactly. Neither is data: they are "no value" markers that survived a truthy +-- guard. Root cause is in the CC statement processors' Parse node, which does +-- foreign_currency_code: t.foreign_currency_code || null +-- and a non-empty string is truthy, so a model that answers with the STRING +-- "null" (rather than JSON null) passes straight through. Same for -1. Fixed +-- workflow-side in the same change; this migration cleans what already landed. +-- Not purely historical -- 155 rows in 2026-03, but also 8 in May, 3 in July +-- and 21 in August 2026. +-- +-- MAPPINGS WERE VERIFIED AGAINST IMPLIED RATE, NOT TRUSTED FROM THE LABEL, +-- because a row whose rate contradicts its claimed currency is a broken ledger +-- row rather than a mislabelled one (the trap order_transaction_bridge's rate +-- guard exists for: 2 of its first 73 matches were broken rows, not bad +-- matches). Median amount/foreign_currency_amount per raw code vs the ISO +-- median over statement rows (n>=5): +-- BAHT 0.0433 vs THB 0.0470 INDIAN RUPEE 0.0183 vs INR 0.0180 +-- U.A.E. DIRHAM 0.4287 vs AED 0.4184 US DOLLAR 1.4054 vs USD 1.5513 +-- NZ DOLLAR 0.8243 vs NZD 0.9311 EURO 1.6342 vs EUR 1.6964 +-- Every label agrees with its rate. foreign_currency_amount is untouched by the +-- relabels -- only the code was ever wrong, so this is a relabel, never a +-- recompute. +-- +-- EXPECTED ROW COUNTS (verified pre-flight, includes superseded rows): +-- code 'null' -> NULL 187 BAHT -> THB 13 +-- amount -1.00 -> NULL 42 U.A.E. DIRHAM -> AED 10 +-- NZ DOLLAR -> NZD 6 US DOLLAR -> USD 4 +-- INDIAN RUPEE -> INR 4 EURO -> EUR 2 +-- Pre-flight also confirmed: no -1.00 amount exists outside the 'null'-code +-- rows, and no other non-ISO code exists. + +BEGIN; + +-- Sentinels first: the -1.00 amounts live only on 'null'-code rows, so clear +-- them before that code is nulled and the set becomes unfindable. +UPDATE transactions SET foreign_currency_amount = NULL + WHERE foreign_currency_amount = -1.00 AND foreign_currency_code = 'null'; + +UPDATE transactions SET foreign_currency_code = NULL + WHERE foreign_currency_code = 'null'; + +-- Word-form relabels. +UPDATE transactions SET foreign_currency_code = 'THB' WHERE foreign_currency_code = 'BAHT'; +UPDATE transactions SET foreign_currency_code = 'AED' WHERE foreign_currency_code = 'U.A.E. DIRHAM'; +UPDATE transactions SET foreign_currency_code = 'NZD' WHERE foreign_currency_code = 'NZ DOLLAR'; +UPDATE transactions SET foreign_currency_code = 'USD' WHERE foreign_currency_code = 'US DOLLAR'; +UPDATE transactions SET foreign_currency_code = 'INR' WHERE foreign_currency_code = 'INDIAN RUPEE'; +UPDATE transactions SET foreign_currency_code = 'EUR' WHERE foreign_currency_code = 'EURO'; + +-- Postcondition: every remaining code is ISO-4217 shaped. Fails the whole +-- transaction rather than leaving a partial normalisation behind. +DO $$ +DECLARE bad int; +BEGIN + SELECT count(*) INTO bad FROM transactions + WHERE foreign_currency_code IS NOT NULL AND foreign_currency_code !~ '^[A-Z]{3}$'; + IF bad <> 0 THEN + RAISE EXCEPTION 'normalisation incomplete: % non-ISO codes remain', bad; + END IF; +END $$; + +COMMIT;