167 lines
8.7 KiB
JavaScript
167 lines
8.7 KiB
JavaScript
#!/usr/bin/env node
|
||
// Build data/airports_world.csv from the full OurAirports world dump (ISS-079, v0.44.0).
|
||
//
|
||
// Pipeline: download → keep `large_airport` rows only (same density curation as the EU file,
|
||
// v0.2.8) → greedy 40 km haversine clustering → one airport per cluster ("the biggest").
|
||
// v0.61.0 (ISS-097): additionally keeps every non-large fixed-wing row that is REMOTE — no other
|
||
// airport within 1000 km of the kept set (lib/airports.ts isRemote) — so Antarctica, the high
|
||
// Arctic, Alaska, northern Canada and far islands are represented even though they carry no
|
||
// large_airport; same-station twins collapse via the same 40 km clustering.
|
||
// The output keeps the exact OurAirports header so lib/airports.ts parses it unchanged.
|
||
//
|
||
// Usage: node scripts/build-world-airports.mjs (writes data/airports_world.csv, prints stats)
|
||
// Aborts without writing anything if the download fails after retries — no fabricated data.
|
||
|
||
import { promises as fs } from 'node:fs';
|
||
import path from 'node:path';
|
||
|
||
const SOURCE_URL = 'https://davidmegginson.github.io/ourairports-data/airports.csv'; // legacy ourairports.com/data URL is dead (see CHANGELOG v0.1.0)
|
||
const OUT_FILE = path.join(import.meta.dirname, '..', 'data', 'airports_world.csv');
|
||
const CLUSTER_RADIUS_KM = 40; // mid of the issue's 30–50 km range — n≈1k rows makes O(n²) trivial
|
||
// v0.61.0 (ISS-097): mirrors REMOTE_RADIUS_KM in lib/airports.ts — "no other airport within this
|
||
// radius" is what makes a small airport worth keeping (Antarctica, high Arctic, far islands).
|
||
const REMOTE_RADIUS_KM = 1000;
|
||
|
||
// Same fixed-wing type set as INCLUDED_TYPES in lib/airports.ts (standalone copy — the script must
|
||
// stay dependency-free): all fixed-wing airports plus seaplane bases.
|
||
const FIXED_WING_TYPES = new Set(['airport', 'small_airport', 'medium_airport', 'large_airport', 'seaplane_base']);
|
||
|
||
// Standalone copy of isRemote in lib/airports.ts, for CSV rows: true when `row` has no member of
|
||
// `others` within REMOTE_RADIUS_KM. Callers pass disjoint sets (the row itself is not in `others`).
|
||
function isRemoteRow(row, others) {
|
||
for (const o of others) {
|
||
if (haversineKm(row, o) <= REMOTE_RADIUS_KM) return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
// --- Minimal RFC-4180 parser (same semantics as lib/airports.ts parseCsv, standalone copy) ---
|
||
function parseCsv(text) {
|
||
const rows = [];
|
||
let row = [];
|
||
let field = '';
|
||
let inQuotes = false;
|
||
for (let i = 0; i < text.length; i++) {
|
||
const c = text[i];
|
||
if (inQuotes) {
|
||
if (c === '"') {
|
||
if (text[i + 1] === '"') { field += '"'; i++; } else inQuotes = false;
|
||
} else field += c;
|
||
} else if (c === '"') inQuotes = true;
|
||
else if (c === ',') { row.push(field); field = ''; }
|
||
else if (c === '\n') { row.push(field); rows.push(row); row = []; field = ''; }
|
||
else if (c !== '\r') field += c;
|
||
}
|
||
if (field.length > 0 || row.length > 0) { row.push(field); rows.push(row); }
|
||
const [header, ...rest] = rows;
|
||
return rest
|
||
.filter((r) => r.length === header.length && !(r.length === 1 && r[0] === ''))
|
||
.map((r) => Object.fromEntries(header.map((h, i) => [h, r[i]])));
|
||
}
|
||
|
||
function haversineKm(a, b) {
|
||
const toRad = (d) => (d * Math.PI) / 180;
|
||
const dLat = toRad(b.latitude_deg - a.latitude_deg);
|
||
const dLon = toRad(b.longitude_deg - a.longitude_deg);
|
||
const x =
|
||
Math.sin(dLat / 2) ** 2 +
|
||
Math.cos(toRad(a.latitude_deg)) * Math.cos(toRad(b.latitude_deg)) * Math.sin(dLon / 2) ** 2;
|
||
return 2 * 6371 * Math.asin(Math.min(1, Math.sqrt(x)));
|
||
}
|
||
|
||
// "Biggest" proxy — the CSV carries no passenger/traffic data, so rank by service signals:
|
||
// scheduled_service first, then an IATA code (commercially served), then a Wikipedia page,
|
||
// then file order. ponytail: good-enough proxy for hub-vs-satellite; swap in pax data if it ever exists.
|
||
function biggestOf(members) {
|
||
const rank = (r) => [r.scheduled_service === 'yes' ? 1 : 0, r.iata_code ? 1 : 0, r.wikipedia_link ? 1 : 0];
|
||
let best = members[0];
|
||
for (const m of members.slice(1)) {
|
||
const rb = rank(m);
|
||
const rk = rank(best);
|
||
if (rb[0] > rk[0] || (rb[0] === rk[0] && rb[1] > rk[1]) || (rb[0] === rk[0] && rb[1] === rk[1] && rb[2] > rk[2])) best = m;
|
||
}
|
||
return best; // ties keep the lowest file order (members are in file order)
|
||
}
|
||
|
||
async function download(url, attempts = 3) {
|
||
for (let i = 1; i <= attempts; i++) {
|
||
try {
|
||
const res = await fetch(url);
|
||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||
const text = await res.text();
|
||
if (!(text.startsWith('"id"') || text.startsWith('id,')) || text.split('\n').length < 10_000) throw new Error('response does not look like the OurAirports CSV');
|
||
return text;
|
||
} catch (err) {
|
||
console.error(`download attempt ${i}/${attempts} failed: ${err.message}`);
|
||
if (i === attempts) throw err;
|
||
await new Promise((r) => setTimeout(r, 2000 * i));
|
||
}
|
||
}
|
||
throw new Error('unreachable');
|
||
}
|
||
|
||
const text = await download(SOURCE_URL);
|
||
const [headerLine] = text.split('\n');
|
||
const header = headerLine.replace(/\r$/, '').split(',').map((h) => h.replace(/^"|"$/g, ''));
|
||
if (header.length !== 19) { console.error(`unexpected header (${header.length} columns): ${headerLine}`); process.exit(1); }
|
||
|
||
const allRows = parseCsv(text);
|
||
// v0.61.0 (ISS-097): curation runs over ALL fixed-wing rows, not just large_airport — the remote
|
||
// pass below needs the full pool to test against.
|
||
const fixed = allRows.filter((r) => FIXED_WING_TYPES.has(r.type));
|
||
const large = fixed.filter((r) => r.type === 'large_airport');
|
||
console.log(`downloaded ${allRows.length} rows; ${fixed.length} are fixed-wing, ${large.length} of them large_airport`);
|
||
|
||
// Greedy clustering in file order: a row joins the first cluster whose anchor is within the
|
||
// radius, otherwise it starts a new one. Deterministic — same input, same output. Returns the
|
||
// "biggest" member (one airport per cluster) plus the clusters themselves for reporting.
|
||
function clusterWinners(rows) {
|
||
const clusters = []; // { anchorIdx (into rows), members }
|
||
for (let i = 0; i < rows.length; i++) {
|
||
let joined = false;
|
||
for (const c of clusters) {
|
||
if (haversineKm(rows[i], rows[c.anchorIdx]) <= CLUSTER_RADIUS_KM) { c.members.push(i); joined = true; break; }
|
||
}
|
||
if (!joined) clusters.push({ anchorIdx: i, members: [i] });
|
||
}
|
||
const winners = clusters.map((c) => biggestOf(c.members.map((i) => rows[i])));
|
||
return { winners, clusters };
|
||
}
|
||
|
||
const { winners: kept, clusters: largeClusters } = clusterWinners(large);
|
||
const multiClusters = [];
|
||
for (const c of largeClusters) {
|
||
if (c.members.length > 1) {
|
||
const winner = biggestOf(c.members.map((i) => large[i]));
|
||
multiClusters.push({
|
||
city: winner.municipality || winner.name,
|
||
keptIcao: winner.icao_code || winner.ident,
|
||
dropped: c.members.map((i) => large[i]).filter((r) => r !== winner).map((r) => `${r.icao_code || r.ident} (${r.name})`),
|
||
});
|
||
}
|
||
}
|
||
|
||
// v0.61.0 (ISS-097): "special" remote airports — every non-large fixed-wing row with no other
|
||
// airport within REMOTE_RADIUS_KM of the kept set joins the dataset, so empty regions get their
|
||
// real (small) airports instead of nothing. Same-station twins (two strips <40 km apart at one
|
||
// base) collapse via the same greedy clustering as above.
|
||
const remote = fixed.filter((r) => r.type !== 'large_airport' && isRemoteRow(r, kept));
|
||
const { winners: remoteKept } = clusterWinners(remote);
|
||
|
||
// --- Guards (fail loud before writing anything) ---
|
||
const finalRows = [...kept, ...remoteKept];
|
||
const key = (r) => r.icao_code || r.ident; // the effective ICAO consumers key by (toGeoJson fallback)
|
||
const keys = new Set(finalRows.map(key));
|
||
if (keys.size !== finalRows.length) { console.error('duplicate ICAO in curated set — aborting'); process.exit(1); }
|
||
for (const must of ['EDDF', 'UNNT', 'LPPD']) { // EDDF is the seed hub; UNNT (Novosibirsk) + LPPD (Ponta Delgada) pin the 409-backstop test pair
|
||
if (!keys.has(must)) { console.error(`required airport ${must} missing from curated set — aborting`); process.exit(1); }
|
||
}
|
||
if (remoteKept.length === 0) { console.error('no remote airports found in the dump — curation rule broken, aborting'); process.exit(1); }
|
||
if (finalRows.length < 300 || finalRows.length > 2500) { console.error(`unexpected row count ${finalRows.length}`); process.exit(1); }
|
||
|
||
await fs.writeFile(OUT_FILE, [headerLine.replace(/\r$/, ''), ...finalRows.map((r) => header.map((h) => `"${String(r[h] ?? '').replace(/"/g, '""')}"`).join(','))].join('\n') + '\n', 'utf8');
|
||
|
||
console.log(`wrote ${OUT_FILE}: ${kept.length} large + ${remoteKept.length} remote = ${finalRows.length} rows (${large.length - kept.length} large dropped into ${multiClusters.length} multi-airport clusters)`);
|
||
console.log('sample merged cities (kept ← dropped):');
|
||
for (const c of multiClusters.slice(0, 15)) console.log(` ${c.city}: kept ${c.keptIcao} ← ${c.dropped.join(', ')}`);
|