World airport dataset + distance cap on generated jobs (ISS-079, v0.44.0)

scripts/build-world-airports.mjs

@@ -0,0 +1,130 @@
+#!/usr/bin/env node
+// Build data/airports_world.csv from the full OurAirports world dump (ISS-079, v0.44.0).
+//
+// Pipeline: download → keep `large_airport` rows only (same density curation as the EU file,
+// v0.2.8) → greedy 40 km haversine clustering → one airport per cluster ("the biggest").
+// The output keeps the exact OurAirports header so lib/airports.ts parses it unchanged.
+//
+// Usage: node scripts/build-world-airports.mjs   (writes data/airports_world.csv, prints stats)
+// Aborts without writing anything if the download fails after retries — no fabricated data.
+
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+
+const SOURCE_URL = 'https://davidmegginson.github.io/ourairports-data/airports.csv'; // legacy ourairports.com/data URL is dead (see CHANGELOG v0.1.0)
+const OUT_FILE = path.join(import.meta.dirname, '..', 'data', 'airports_world.csv');
+const CLUSTER_RADIUS_KM = 40; // mid of the issue's 30–50 km range — n≈1k rows makes O(n²) trivial
+
+// --- Minimal RFC-4180 parser (same semantics as lib/airports.ts parseCsv, standalone copy) ---
+function parseCsv(text) {
+  const rows = [];
+  let row = [];
+  let field = '';
+  let inQuotes = false;
+  for (let i = 0; i < text.length; i++) {
+    const c = text[i];
+    if (inQuotes) {
+      if (c === '"') {
+        if (text[i + 1] === '"') { field += '"'; i++; } else inQuotes = false;
+      } else field += c;
+    } else if (c === '"') inQuotes = true;
+    else if (c === ',') { row.push(field); field = ''; }
+    else if (c === '\n') { row.push(field); rows.push(row); row = []; field = ''; }
+    else if (c !== '\r') field += c;
+  }
+  if (field.length > 0 || row.length > 0) { row.push(field); rows.push(row); }
+  const [header, ...rest] = rows;
+  return rest
+    .filter((r) => r.length === header.length && !(r.length === 1 && r[0] === ''))
+    .map((r) => Object.fromEntries(header.map((h, i) => [h, r[i]])));
+}
+
+function haversineKm(a, b) {
+  const toRad = (d) => (d * Math.PI) / 180;
+  const dLat = toRad(b.latitude_deg - a.latitude_deg);
+  const dLon = toRad(b.longitude_deg - a.longitude_deg);
+  const x =
+    Math.sin(dLat / 2) ** 2 +
+    Math.cos(toRad(a.latitude_deg)) * Math.cos(toRad(b.latitude_deg)) * Math.sin(dLon / 2) ** 2;
+  return 2 * 6371 * Math.asin(Math.min(1, Math.sqrt(x)));
+}
+
+// "Biggest" proxy — the CSV carries no passenger/traffic data, so rank by service signals:
+// scheduled_service first, then an IATA code (commercially served), then a Wikipedia page,
+// then file order. ponytail: good-enough proxy for hub-vs-satellite; swap in pax data if it ever exists.
+function biggestOf(members) {
+  const rank = (r) => [r.scheduled_service === 'yes' ? 1 : 0, r.iata_code ? 1 : 0, r.wikipedia_link ? 1 : 0];
+  let best = members[0];
+  for (const m of members.slice(1)) {
+    const rb = rank(m);
+    const rk = rank(best);
+    if (rb[0] > rk[0] || (rb[0] === rk[0] && rb[1] > rk[1]) || (rb[0] === rk[0] && rb[1] === rk[1] && rb[2] > rk[2])) best = m;
+  }
+  return best; // ties keep the lowest file order (members are in file order)
+}
+
+async function download(url, attempts = 3) {
+  for (let i = 1; i <= attempts; i++) {
+    try {
+      const res = await fetch(url);
+      if (!res.ok) throw new Error(`HTTP ${res.status}`);
+      const text = await res.text();
+      if (!(text.startsWith('"id"') || text.startsWith('id,')) || text.split('\n').length < 10_000) throw new Error('response does not look like the OurAirports CSV');
+      return text;
+    } catch (err) {
+      console.error(`download attempt ${i}/${attempts} failed: ${err.message}`);
+      if (i === attempts) throw err;
+      await new Promise((r) => setTimeout(r, 2000 * i));
+    }
+  }
+  throw new Error('unreachable');
+}
+
+const text = await download(SOURCE_URL);
+const [headerLine] = text.split('\n');
+const header = headerLine.replace(/\r$/, '').split(',').map((h) => h.replace(/^"|"$/g, ''));
+if (header.length !== 19) { console.error(`unexpected header (${header.length} columns): ${headerLine}`); process.exit(1); }
+
+const allRows = parseCsv(text);
+const large = allRows.filter((r) => r.type === 'large_airport');
+console.log(`downloaded ${allRows.length} rows; ${large.length} are large_airport`);
+
+// Greedy clustering in file order: a row joins the first cluster whose anchor is within the
+// radius, otherwise it starts a new one. Deterministic — same input, same output.
+const clusters = []; // { anchorIdx (into `large`), members }
+for (let i = 0; i < large.length; i++) {
+  let joined = false;
+  for (const c of clusters) {
+    if (haversineKm(large[i], large[c.anchorIdx]) <= CLUSTER_RADIUS_KM) { c.members.push(i); joined = true; break; }
+  }
+  if (!joined) clusters.push({ anchorIdx: i, members: [i] });
+}
+
+const kept = [];
+const multiClusters = [];
+for (const c of clusters) {
+  const winner = biggestOf(c.members.map((i) => large[i]));
+  kept.push(winner);
+  if (c.members.length > 1) {
+    multiClusters.push({
+      city: winner.municipality || winner.name,
+      keptIcao: winner.icao_code || winner.ident,
+      dropped: c.members.map((i) => large[i]).filter((r) => r !== winner).map((r) => `${r.icao_code || r.ident} (${r.name})`),
+    });
+  }
+}
+
+// --- Guards (fail loud before writing anything) ---
+const key = (r) => r.icao_code || r.ident; // the effective ICAO consumers key by (toGeoJson fallback)
+const keys = new Set(kept.map(key));
+if (keys.size !== kept.length) { console.error('duplicate ICAO in curated set — aborting'); process.exit(1); }
+for (const must of ['EDDF', 'UNNT', 'LPPD']) { // EDDF is the seed hub; UNNT (Novosibirsk) + LPPD (Ponta Delgada) pin the 409-backstop test pair
+  if (!keys.has(must)) { console.error(`required airport ${must} missing from curated set — aborting`); process.exit(1); }
+}
+if (kept.length < 300 || kept.length > 2500) { console.error(`unexpected row count ${kept.length}`); process.exit(1); }
+
+await fs.writeFile(OUT_FILE, [headerLine.replace(/\r$/, ''), ...kept.map((r) => header.map((h) => `"${String(r[h] ?? '').replace(/"/g, '""')}"`).join(','))].join('\n') + '\n', 'utf8');
+
+console.log(`wrote ${OUT_FILE}: ${kept.length} rows kept, ${large.length - kept.length} dropped into ${multiClusters.length} multi-airport clusters`);
+console.log('sample merged cities (kept ← dropped):');
+for (const c of multiClusters.slice(0, 15)) console.log(`  ${c.city}: kept ${c.keptIcao} ← ${c.dropped.join(', ')}`);