v0.28.0 (#46): per-region stats — area×population-weighted-median Nation.stats (PR 1 of 2)

scripts/build-region-geo.mjs

@@ -0,0 +1,165 @@
+/**
+ * scripts/build-region-geo.mjs — build the region **geo** manifest:
+ * `public/region-geo.json` = `{ "<regionKey>": { "area": <number>, "population": <number> }, ... }`.
+ *
+ * For every sub-region (region key, see `public/region-keys.json`) this records
+ * two facts the v0.28 per-region stats model (and its area×population weighting)
+ * needs:
+ *   - `area`       — the region's **total** surface area in km², from
+ *                    `@turf/area` on the FINAL generated feature
+ *                    (`public/regions.geo.json`) — turf reports square metres,
+ *                    so the raw value is divided by 1e6 — rounded to 2 decimals.
+ *   - `population` — the region's share of its **parent country's** real
+ *                    population (Natural Earth 1:10m admin-0 `POP_EST`, from the
+ *                    committed `public/ne-admin0-pop.json`), distributed across
+ *                    the country's sub-regions **proportional to each region's
+ *                    area** (a first-order "uniform population density within a
+ *                    country" approximation — Natural Earth admin-1, which the
+ *                    region generator uses, carries no per-unit population).
+ *
+ * Population distribution uses **largest remainder** per country, so the sum of
+ * a country's sub-region populations is EXACTLY the country's POP_EST, and the
+ * sum over all 1,800 regions is EXACTLY the sum of the 177 repo countries'
+ * POP_EST (7,654,092,021) — no rounding drift.
+ *
+ * Method — plain turf on the pre-generated geo file (turf is allowed in .mjs
+ * scripts; it is deliberately kept OUT of the TS typecheck program, see the
+ * v0.17 note). The population source is the COMMITTED `public/ne-admin0-pop.json`
+ * (a small derived table produced once by `scripts/extract-admin0-pop.mjs` from
+ * the 13 MB Natural Earth admin-0 file, which we do NOT commit) — so this build
+ * is fully reproducible in CI.
+ *
+ * Output contract (asserted by scripts/region-geo.test.ts):
+ *   - one entry for **every** region key, keyed identically to
+ *     `public/region-keys.json` (same key set, 1,800 entries);
+ *   - `area` is a finite number >= 0 (km², <= 2 decimals);
+ *   - `population` is a non-negative integer;
+ *   - the total population across all entries is a non-negative integer.
+ *
+ * Deterministic + idempotent: fixed inputs (the geo file, the key manifest, and
+ * the committed population table), pure turf, largest-remainder with a
+ * (remainder desc, key asc) tie-break, and stable sorted key order → a re-run
+ * rewrites a byte-identical file (the test re-runs this script and compares
+ * bytes).
+ *
+ * Usage: `node scripts/build-region-geo.mjs`  (or `npm run build:region-geo`).
+ */
+import { readFileSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import area from '@turf/area';
+import { feature } from '@turf/helpers';
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const root = path.resolve(here, '..');
+
+const round2 = (v) => Math.round(v * 100) / 100;
+
+// --- load the inputs (plain parse) ------------------------------------------
+const geoPath = path.join(root, 'public', 'regions.geo.json');
+const keysPath = path.join(root, 'public', 'region-keys.json');
+const popPath = path.join(root, 'public', 'ne-admin0-pop.json');
+
+let geo, idToKey, popByCountry;
+try {
+  geo = JSON.parse(readFileSync(geoPath, 'utf8'));
+  idToKey = JSON.parse(readFileSync(keysPath, 'utf8'));
+  popByCountry = JSON.parse(readFileSync(popPath, 'utf8'));
+} catch (err) {
+  console.error(
+    `[build-region-geo] could not load the inputs (${geoPath}, ${keysPath}, ${popPath}). ` +
+      `Is public/ne-admin0-pop.json present? Regenerate it with scripts/extract-admin0-pop.mjs.`,
+  );
+  throw err;
+}
+
+/**
+ * One pass over the geo features: for each region key record its (unrounded)
+ * turf area and its parent ISO_A3, then group the keys by parent so the
+ * population can be distributed per country.
+ */
+const regionArea = new Map(); // key -> unrounded area (km²)
+const regionParent = new Map(); // key -> parent ISO_A3
+for (const f of geo.features ?? []) {
+  const id = f.id == null ? '' : String(f.id);
+  const key = idToKey[id];
+  if (!key) continue; // a geo feature with no manifest entry is skipped
+  // @turf/area reports square METRES; the manifest stores km².
+  const a = area(feature(f.geometry)) / 1e6;
+  regionArea.set(key, a);
+  regionParent.set(key, typeof f.properties?.parent === 'string' ? f.properties.parent : '');
+}
+
+// Group region keys by parent country (file order is stable, but we sort for
+// deterministic population distribution).
+const keysByCountry = new Map(); // parent -> string[] of keys
+for (const [key, parent] of regionParent) {
+  if (!parent) continue; // a region with no parent has no population source
+  const list = keysByCountry.get(parent);
+  if (list) list.push(key);
+  else keysByCountry.set(parent, [key]);
+}
+
+const population = new Map(); // key -> integer population
+for (const [parent, keys] of keysByCountry) {
+  keys.sort(); // deterministic ordering for the tie-break
+  const P = Number.isFinite(popByCountry[parent]) ? Math.max(0, popByCountry[parent]) : 0;
+  const totalArea = keys.reduce((s, k) => s + (regionArea.get(k) || 0), 0);
+  if (P <= 0 || totalArea <= 0) {
+    for (const k of keys) population.set(k, 0);
+    continue;
+  }
+  // Largest remainder: floor each share, then hand out the leftover 1s to the
+  // regions with the largest fractional remainders (ties broken by key asc).
+  const shares = keys.map((k) => {
+    const a = regionArea.get(k) || 0;
+    const ideal = (P * a) / totalArea;
+    return { k, floor: Math.floor(ideal), rem: ideal - Math.floor(ideal) };
+  });
+  const distributed = shares.reduce((s, x) => s + x.floor, 0);
+  let deficit = P - distributed; // 0 <= deficit < keys.length
+  const order = shares
+    .map((x, i) => ({ i, rem: x.rem, k: x.k }))
+    .sort((a, b) => (b.rem - a.rem) || (a.k < b.k ? -1 : a.k > b.k ? 1 : 0));
+  for (let i = 0; i < order.length && deficit > 0; i++) {
+    shares[order[i].i].floor += 1;
+    deficit -= 1;
+  }
+  for (const x of shares) population.set(x.k, x.floor);
+}
+
+// --- assemble the manifest (sorted keys) -------------------------------------
+const expectedKeys = Object.values(idToKey);
+const manifest = {};
+for (const k of [...expectedKeys].sort()) {
+  const a = regionArea.get(k);
+  const p = population.get(k);
+  if (a === undefined || p === undefined) {
+    throw new Error(`region key ${JSON.stringify(k)} has no computed area/population`);
+  }
+  manifest[k] = { area: round2(a), population: p };
+}
+
+// --- hard assertions (the test re-checks these against the committed file) ----
+if (Object.keys(manifest).length !== expectedKeys.length) {
+  throw new Error(
+    `expected ${expectedKeys.length} region-geo entries (one per region key), got ${Object.keys(manifest).length}`,
+  );
+}
+let totalPop = 0;
+let zeros = 0;
+for (const [k, v] of Object.entries(manifest)) {
+  if (!Number.isFinite(v.area) || v.area < 0) throw new Error(`bad area for ${k}: ${JSON.stringify(v)}`);
+  if (!Number.isInteger(v.population) || v.population < 0) throw new Error(`bad population for ${k}: ${JSON.stringify(v)}`);
+  if (v.area === 0) zeros += 1;
+  totalPop += v.population;
+}
+
+// --- write idempotently (sorted keys, 2-space indent, trailing newline) ------
+writeFileSync(path.join(root, 'public', 'region-geo.json'), JSON.stringify(manifest, null, 2) + '\n');
+
+// --- summary ------------------------------------------------------------------
+console.log(
+  `[build-region-geo] wrote public/region-geo.json — ${Object.keys(manifest).length} region keys` +
+    `${zeros ? ` (${zeros} zero-area regions)` : ''}, total population ${totalPop.toLocaleString('en-US')}`,
+);

scripts/extract-admin0-pop.mjs

@@ -0,0 +1,78 @@
+/**
+ * scripts/extract-admin0-pop.mjs — one-time extraction of the population
+ * SOURCE for the region-geo manifest: `public/ne-admin0-pop.json`.
+ *
+ * Output = `{ "<ADM0_A3>": <POP_EST>, ... }` — the estimated population
+ * (POP_EST) of every Natural Earth 1:10m **admin-0 country** code, keeping
+ * the **maximum** POP_EST seen for each code (Natural Earth ships a few
+ * duplicated / broken-out features per code, e.g. the `TLC` stubs, whose
+ * POP_EST is 0; the real country always wins by having the larger value).
+ * Only codes with a finite, positive POP_EST are recorded (a 0/null POP_EST
+ * has nothing to distribute). Keys are sorted; the file is written with
+ * 2-space indent + trailing newline (deterministic, committed).
+ *
+ * Why a separate committed file: `public/region-geo.json` (built by
+ * `build-region-geo.mjs`) needs a per-country population, but the raw
+ * Natural Earth GeoJSON (13 MB) is a *source* file we deliberately do NOT
+ * commit (see the AGENTS.md data-pipeline note). This script distils it
+ * into a tiny (a few KB) derived table that IS committed, so the region-geo
+ * builder stays fully reproducible in CI without the source present.
+ * Re-run only when the underlying Natural Earth data changes.
+ *
+ * The 1:10m admin-1 file (the one the region generator uses) carries NO
+ * population; the 1:10m admin-0 file does (uppercase `POP_EST` / `ADM0_A3`).
+ *
+ * Usage:
+ *   NE_ADMIN0=/path/to/ne_10m_admin_0_countries.geojson node scripts/extract-admin0-pop.mjs
+ * (default source: /tmp/ne10m_admin0.geojson)
+ */
+import { readFileSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const root = path.resolve(here, '..');
+
+const src = process.env.NE_ADMIN0 || '/tmp/ne10m_admin0.geojson';
+let raw;
+try {
+  raw = readFileSync(src, 'utf8');
+} catch (err) {
+  console.error(
+    `[extract-admin0-pop] could not read the Natural Earth admin-0 source at ${src}. ` +
+      `Download it (ne_10m_admin_0_countries.geojson) and re-run with NE_ADMIN0 pointing at it.`,
+  );
+  throw err;
+}
+
+const geo = JSON.parse(raw);
+
+/** ADM0_A3 -> max finite, positive POP_EST. */
+const byCode = new Map();
+for (const f of geo.features ?? []) {
+  const p = f.properties ?? {};
+  const code = typeof p.ADM0_A3 === 'string' ? p.ADM0_A3 : '';
+  const pop = typeof p.POP_EST === 'number' ? p.POP_EST : NaN;
+  if (!code || !Number.isFinite(pop) || pop <= 0) continue;
+  const prev = byCode.get(code);
+  if (prev === undefined || pop > prev) byCode.set(code, Math.round(pop));
+}
+
+if (byCode.size === 0) {
+  throw new Error('[extract-admin0-pop] no finite positive POP_EST found in the source — wrong file?');
+}
+
+// Deterministic: sorted keys, 2-space indent, trailing newline.
+const keys = [...byCode.keys()].sort();
+const manifest = {};
+for (const k of keys) manifest[k] = byCode.get(k);
+
+const out = path.join(root, 'public', 'ne-admin0-pop.json');
+writeFileSync(out, JSON.stringify(manifest, null, 2) + '\n');
+
+let sum = 0;
+for (const v of byCode.values()) sum += v;
+console.log(
+  `[extract-admin0-pop] wrote public/ne-admin0-pop.json — ${byCode.size} country codes, ` +
+    `total population ${sum.toLocaleString('en-US')} (Natural Earth 1:10m admin-0, max POP_EST per code)`,
+);

scripts/region-geo.test.ts

@@ -0,0 +1,139 @@
+/**
+ * scripts/region-geo.test.ts — validation for the committed
+ * `public/region-geo.json` (built by `scripts/build-region-geo.mjs`).
+ *
+ * The manifest maps each region key to its { area (km²), population } — the
+ * weighting source for the per-region stats model (v0.28, #46): a nation's
+ * four 0–100 stats are derived as the area×population-weighted median of the
+ * stats of the regions it owns. It is a load-bearing data artifact — this
+ * test pins its contract.
+ *
+ * The `area` is the turf area of the region's final geometry (km², 2 dp); the
+ * `population` is the country's Natural-Earth admin-0 `POP_EST` (from the
+ * committed `public/ne-admin0-pop.json`) distributed across its sub-regions
+ * proportional to area via largest-remainder, so every country's sub-regions
+ * sum EXACTLY to the country's total (and the global total is the sum over the
+ * 177 repo countries). This is a dependency-free test: it reads the JSON with
+ * plain `fs`/`JSON.parse` (no `@turf`/geojson types, which would pull
+ * `@types/geojson` into the tsc program and break `WorldMap.tsx` — see the
+ * v0.17 note). The determinism test re-runs the (turf `.mjs`) builder and
+ * compares bytes, matching that the builder is idempotent.
+ */
+import { readFileSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { describe, expect, it } from 'vitest';
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const root = path.resolve(here, '..');
+
+function readJson(rel: string): Record<string, unknown> {
+  return JSON.parse(readFileSync(path.join(root, rel), 'utf8'));
+}
+
+interface GeoEntry {
+  area: number;
+  population: number;
+}
+
+// The 177 repo countries' admin-0 population totals (the committed derived
+// source). Only the countries actually present in the region dataset appear.
+const admin0 = readJson('public/ne-admin0-pop.json') as Record<string, number>;
+// feature id -> region key
+const manifest = readJson('public/region-keys.json') as Record<string, string>;
+// feature id -> { name, parent (ISO_A3) }
+const regGeo = readJson('public/regions.geo.json') as {
+  features?: Array<{ id?: string | number; properties?: { name?: string; parent?: string } }>;
+};
+
+// region key -> parent ISO_A3 (via the key manifest + the geo feature parent).
+const keyToParent: Record<string, string> = {};
+for (const f of regGeo.features ?? []) {
+  const key = manifest[String(f.id)];
+  const parent = f.properties?.parent;
+  if (key && parent) keyToParent[key] = parent;
+}
+
+const geo = readJson('public/region-geo.json') as Record<string, GeoEntry>;
+const keys = Object.values(manifest);
+
+describe('region-geo.json', () => {
+  it('has exactly one entry per region key (same key set as region-keys.json)', () => {
+    expect(new Set(Object.keys(geo))).toEqual(new Set(keys));
+    expect(Object.keys(geo)).toHaveLength(1800);
+  });
+
+  it('is keyed in sorted (deterministic) order', () => {
+    const ks = Object.keys(geo);
+    expect(ks).toEqual([...ks].sort());
+  });
+
+  it('every entry is a finite { area ≥ 0 with ≤ 2 dp, integer population ≥ 0 }', () => {
+    for (const [k, v] of Object.entries(geo)) {
+      expect(Number.isFinite(v.area), `non-finite area for ${k}`).toBe(true);
+      expect(Number.isFinite(v.population), `non-finite population for ${k}`).toBe(true);
+      expect(v.area >= 0, `negative area for ${k}: ${v.area}`).toBe(true);
+      // area is recorded to at most 2 decimal places (km²)
+      expect(Math.round(v.area * 100) / 100 === v.area, `area over 2dp for ${k}: ${v.area}`).toBe(true);
+      expect(Number.isInteger(v.population), `non-integer population for ${k}: ${v.population}`).toBe(true);
+      expect(v.population >= 0, `negative population for ${k}: ${v.population}`).toBe(true);
+    }
+  });
+
+  it('the global population total equals the sum of the 177 repo countries\' admin-0 totals', () => {
+    // The distribution preserved each country's total exactly, so the grand
+    // total over all sub-regions is the sum over the repo countries only.
+    const total = Object.values(geo).reduce((s, v) => s + v.population, 0);
+    expect(total).toBe(7654092021);
+  });
+
+  it('preserves each country\'s population total across its sub-regions (largest-remainder)', () => {
+    // For every parent country present in the region dataset, the sum of its
+    // sub-regions' populations must equal that country's admin-0 POP_EST
+    // EXACTLY — this is the invariant the largest-remainder distribution
+    // guarantees (no population is created or lost by the split).
+    const byParent: Record<string, number> = {};
+    for (const [k, v] of Object.entries(geo)) {
+      const p = keyToParent[k];
+      if (!p) throw new Error(`no parent for region key ${k}`);
+      byParent[p] = (byParent[p] ?? 0) + v.population;
+    }
+    expect(Object.keys(byParent).length).toBe(177); // the 177 repo countries
+    for (const [p, sum] of Object.entries(byParent)) {
+      expect(admin0[p], `missing admin-0 population for ${p}`).toBeDefined();
+      expect(sum, `population drift for ${p}: sub-regions ${sum} vs admin-0 ${admin0[p]}`).toBe(admin0[p]);
+    }
+  });
+
+  it('spot-checks well-known regions for exact area + population', () => {
+    const is = (k: string, area: number, population: number) => {
+      const v = geo[k];
+      expect(v, `missing geo for ${k}`).toBeDefined();
+      expect(v!.area, `area for ${k}`).toBe(area);
+      expect(v!.population, `population for ${k}`).toBe(population);
+    };
+    is('california-kikh', 413329.21, 14184195); // US-CA
+    is('arkhangel-sk-lhbk', 305257.7, 2579108); // RU-ARK mainland core
+    is('pechora-qwhs', 37399.76, 315989); // RU-PEO (Arctic-coast band, #41)
+    is('yamal-gosw', 50598.25, 427502); // RU-YAM (Arctic-coast band, #41)
+    is('new-earth-eikt', 15721.27, 132828); // RU-ARK-NE (island group, #31)
+    is('severnaya-zemlya-awqp', 1209.93, 10223); // RU-ARK-SZ (island group, #31)
+    is('ain-lqtt', 22557.77, 2304126); // FR-01 (Ain)
+  });
+
+  it('is deterministic: re-running the builder rewrites a byte-identical file', () => {
+    const file = path.join(root, 'public', 'region-geo.json');
+    const before = readFileSync(file, 'utf8');
+    // The builder is cwd-independent (it resolves the repo root from its own
+    // location) and idempotent: same input files (regions.geo.json +
+    // region-keys.json + ne-admin0-pop.json) + pure turf + fixed 2-dp rounding
+    // + stable (sorted) key order → identical bytes.
+    execFileSync(process.execPath, [path.join(root, 'scripts', 'build-region-geo.mjs')], {
+      cwd: root,
+      stdio: 'pipe',
+    });
+    const after = readFileSync(file, 'utf8');
+    expect(after).toBe(before);
+  });
+});