Flight-Ops/scripts/build-world-airports.mjs

#!/usr/bin/env node
// Build data/airports_world.csv from the full OurAirports world dump (ISS-079, v0.44.0).
//
// Pipeline: download → keep `large_airport` rows only (same density curation as the EU file,
// v0.2.8) → greedy 40 km haversine clustering → one airport per cluster ("the biggest").
// v0.68.0 (ISS-144): additionally keeps a sparse set of "special" remote airports — non-large
// fixed-wing rows with no large airport within 1000 km, greedily thinned to one per ~1000 km
// region (scheduled service + larger types preferred) so Antarctica, the high Arctic, Alaska,
// northern Canada and far islands are represented without over-filling empty regions.
// The output keeps the exact OurAirports header so lib/airports.ts parses it unchanged.
//
// Usage: node scripts/build-world-airports.mjs   (writes data/airports_world.csv, prints stats)
// Aborts without writing anything if the download fails after retries — no fabricated data.

import { promises as fs } from 'node:fs';
import path from 'node:path';

const SOURCE_URL = 'https://davidmegginson.github.io/ourairports-data/airports.csv'; // legacy ourairports.com/data URL is dead (see CHANGELOG v0.1.0)
const OUT_FILE = path.join(import.meta.dirname, '..', 'data', 'airports_world.csv');
const CLUSTER_RADIUS_KM = 40; // mid of the issue's 30–50 km range — n≈1k rows makes O(n²) trivial
// v0.61.0 (ISS-097): mirrors REMOTE_RADIUS_KM in lib/airports.ts — "no other airport within this
// radius" is what makes a small airport worth keeping (Antarctica, high Arctic, far islands).
const REMOTE_RADIUS_KM = 1000;

// Same fixed-wing type set as INCLUDED_TYPES in lib/airports.ts (standalone copy — the script must
// stay dependency-free): all fixed-wing airports plus seaplane bases.
const FIXED_WING_TYPES = new Set(['airport', 'small_airport', 'medium_airport', 'large_airport', 'seaplane_base']);

// Standalone copy of isRemote in lib/airports.ts, for CSV rows: true when `row` has no member of
// `others` within REMOTE_RADIUS_KM. Callers pass disjoint sets (the row itself is not in `others`).
function isRemoteRow(row, others) {
  for (const o of others) {
    if (haversineKm(row, o) <= REMOTE_RADIUS_KM) return false;
  }
  return true;
}

// --- Minimal RFC-4180 parser (same semantics as lib/airports.ts parseCsv, standalone copy) ---
function parseCsv(text) {
  const rows = [];
  let row = [];
  let field = '';
  let inQuotes = false;
  for (let i = 0; i < text.length; i++) {
    const c = text[i];
    if (inQuotes) {
      if (c === '"') {
        if (text[i + 1] === '"') { field += '"'; i++; } else inQuotes = false;
      } else field += c;
    } else if (c === '"') inQuotes = true;
    else if (c === ',') { row.push(field); field = ''; }
    else if (c === '\n') { row.push(field); rows.push(row); row = []; field = ''; }
    else if (c !== '\r') field += c;
  }
  if (field.length > 0 || row.length > 0) { row.push(field); rows.push(row); }
  const [header, ...rest] = rows;
  return rest
    .filter((r) => r.length === header.length && !(r.length === 1 && r[0] === ''))
    .map((r) => Object.fromEntries(header.map((h, i) => [h, r[i]])));
}

function haversineKm(a, b) {
  const toRad = (d) => (d * Math.PI) / 180;
  const dLat = toRad(b.latitude_deg - a.latitude_deg);
  const dLon = toRad(b.longitude_deg - a.longitude_deg);
  const x =
    Math.sin(dLat / 2) ** 2 +
    Math.cos(toRad(a.latitude_deg)) * Math.cos(toRad(b.latitude_deg)) * Math.sin(dLon / 2) ** 2;
  return 2 * 6371 * Math.asin(Math.min(1, Math.sqrt(x)));
}

// "Biggest" proxy — the CSV carries no passenger/traffic data, so rank by service signals:
// scheduled_service first, then an IATA code (commercially served), then a Wikipedia page,
// then file order. ponytail: good-enough proxy for hub-vs-satellite; swap in pax data if it ever exists.
function biggestOf(members) {
  const rank = (r) => [r.scheduled_service === 'yes' ? 1 : 0, r.iata_code ? 1 : 0, r.wikipedia_link ? 1 : 0];
  let best = members[0];
  for (const m of members.slice(1)) {
    const rb = rank(m);
    const rk = rank(best);
    if (rb[0] > rk[0] || (rb[0] === rk[0] && rb[1] > rk[1]) || (rb[0] === rk[0] && rb[1] === rk[1] && rb[2] > rk[2])) best = m;
  }
  return best; // ties keep the lowest file order (members are in file order)
}

async function download(url, attempts = 3) {
  for (let i = 1; i <= attempts; i++) {
    try {
      const res = await fetch(url);
      if (!res.ok) throw new Error(`HTTP ${res.status}`);
      const text = await res.text();
      if (!(text.startsWith('"id"') || text.startsWith('id,')) || text.split('\n').length < 10_000) throw new Error('response does not look like the OurAirports CSV');
      return text;
    } catch (err) {
      console.error(`download attempt ${i}/${attempts} failed: ${err.message}`);
      if (i === attempts) throw err;
      await new Promise((r) => setTimeout(r, 2000 * i));
    }
  }
  throw new Error('unreachable');
}

const text = await download(SOURCE_URL);
const [headerLine] = text.split('\n');
const header = headerLine.replace(/\r$/, '').split(',').map((h) => h.replace(/^"|"$/g, ''));
if (header.length !== 19) { console.error(`unexpected header (${header.length} columns): ${headerLine}`); process.exit(1); }

const allRows = parseCsv(text);
// v0.61.0 (ISS-097): curation runs over ALL fixed-wing rows, not just large_airport — the remote
// pass below needs the full pool to test against.
const fixed = allRows.filter((r) => FIXED_WING_TYPES.has(r.type));
const large = fixed.filter((r) => r.type === 'large_airport');
console.log(`downloaded ${allRows.length} rows; ${fixed.length} are fixed-wing, ${large.length} of them large_airport`);

// Greedy clustering in file order: a row joins the first cluster whose anchor is within the
// radius, otherwise it starts a new one. Deterministic — same input, same output. Returns the
// "biggest" member (one airport per cluster) plus the clusters themselves for reporting.
function clusterWinners(rows) {
  const clusters = []; // { anchorIdx (into rows), members }
  for (let i = 0; i < rows.length; i++) {
    let joined = false;
    for (const c of clusters) {
      if (haversineKm(rows[i], rows[c.anchorIdx]) <= CLUSTER_RADIUS_KM) { c.members.push(i); joined = true; break; }
    }
    if (!joined) clusters.push({ anchorIdx: i, members: [i] });
  }
  const winners = clusters.map((c) => biggestOf(c.members.map((i) => rows[i])));
  return { winners, clusters };
}

const { winners: kept, clusters: largeClusters } = clusterWinners(large);
const multiClusters = [];
for (const c of largeClusters) {
  if (c.members.length > 1) {
    const winner = biggestOf(c.members.map((i) => large[i]));
    multiClusters.push({
      city: winner.municipality || winner.name,
      keptIcao: winner.icao_code || winner.ident,
      dropped: c.members.map((i) => large[i]).filter((r) => r !== winner).map((r) => `${r.icao_code || r.ident} (${r.name})`),
    });
  }
}

// v0.68.0 (ISS-144): "special" remote airports — one per ~REMOTE_RADIUS_KM region. Candidates are
// the non-large fixed-wing rows with no large airport within REMOTE_RADIUS_KM (unchanged), sorted
// by priority (scheduled service, then type rank medium < small < seaplane, then numeric id) and
// accepted greedily: a candidate joins only if it is at least REMOTE_RADIUS_KM from EVERY already
// accepted special — no special airport within 1000 km of another. One per ~1000 km region, so
// central Australia / Canada edges get a handful of well-chosen strips instead of dozens; this
// subsumes the old 40 km remote clustering (deleted).
const TYPE_RANK = { medium_airport: 0, small_airport: 1, seaplane_base: 2 }; // legacy 'airport' ranks last
const candidates = fixed.filter((r) => r.type !== 'large_airport' && isRemoteRow(r, kept));
candidates.sort(
  (a, b) =>
    Number(b.scheduled_service === 'yes') - Number(a.scheduled_service === 'yes') ||
    (TYPE_RANK[a.type] ?? 3) - (TYPE_RANK[b.type] ?? 3) ||
    Number(a.id) - Number(b.id),
);
const remoteKept = [];
for (const c of candidates) {
  if (remoteKept.every((k) => haversineKm(c, k) >= REMOTE_RADIUS_KM)) remoteKept.push(c);
}

// --- Guards (fail loud before writing anything) ---
const finalRows = [...kept, ...remoteKept];
const key = (r) => r.icao_code || r.ident; // the effective ICAO consumers key by (toGeoJson fallback)
const keys = new Set(finalRows.map(key));
if (keys.size !== finalRows.length) { console.error('duplicate ICAO in curated set — aborting'); process.exit(1); }
for (const must of ['EDDF', 'UNNT', 'LPPD']) { // EDDF is the seed hub; UNNT (Novosibirsk) + LPPD (Ponta Delgada) pin the 409-backstop test pair
  if (!keys.has(must)) { console.error(`required airport ${must} missing from curated set — aborting`); process.exit(1); }
}
if (remoteKept.length === 0) { console.error('no remote airports found in the dump — curation rule broken, aborting'); process.exit(1); }
if (finalRows.length < 300 || finalRows.length > 2500) { console.error(`unexpected row count ${finalRows.length}`); process.exit(1); }

await fs.writeFile(OUT_FILE, [headerLine.replace(/\r$/, ''), ...finalRows.map((r) => header.map((h) => `"${String(r[h] ?? '').replace(/"/g, '""')}"`).join(','))].join('\n') + '\n', 'utf8');

console.log(`wrote ${OUT_FILE}: ${kept.length} large + ${remoteKept.length} remote = ${finalRows.length} rows (${large.length - kept.length} large dropped into ${multiClusters.length} multi-airport clusters)`);
console.log('sample merged cities (kept ← dropped):');
for (const c of multiClusters.slice(0, 15)) console.log(`  ${c.city}: kept ${c.keptIcao} ← ${c.dropped.join(', ')}`);