131 lines
6.2 KiB
JavaScript
131 lines
6.2 KiB
JavaScript
#!/usr/bin/env node
|
||
// Build data/airports_world.csv from the full OurAirports world dump (ISS-079, v0.44.0).
|
||
//
|
||
// Pipeline: download → keep `large_airport` rows only (same density curation as the EU file,
|
||
// v0.2.8) → greedy 40 km haversine clustering → one airport per cluster ("the biggest").
|
||
// The output keeps the exact OurAirports header so lib/airports.ts parses it unchanged.
|
||
//
|
||
// Usage: node scripts/build-world-airports.mjs (writes data/airports_world.csv, prints stats)
|
||
// Aborts without writing anything if the download fails after retries — no fabricated data.
|
||
|
||
import { promises as fs } from 'node:fs';
|
||
import path from 'node:path';
|
||
|
||
const SOURCE_URL = 'https://davidmegginson.github.io/ourairports-data/airports.csv'; // legacy ourairports.com/data URL is dead (see CHANGELOG v0.1.0)
|
||
const OUT_FILE = path.join(import.meta.dirname, '..', 'data', 'airports_world.csv');
|
||
const CLUSTER_RADIUS_KM = 40; // mid of the issue's 30–50 km range — n≈1k rows makes O(n²) trivial
|
||
|
||
// --- Minimal RFC-4180 parser (same semantics as lib/airports.ts parseCsv, standalone copy) ---
|
||
function parseCsv(text) {
|
||
const rows = [];
|
||
let row = [];
|
||
let field = '';
|
||
let inQuotes = false;
|
||
for (let i = 0; i < text.length; i++) {
|
||
const c = text[i];
|
||
if (inQuotes) {
|
||
if (c === '"') {
|
||
if (text[i + 1] === '"') { field += '"'; i++; } else inQuotes = false;
|
||
} else field += c;
|
||
} else if (c === '"') inQuotes = true;
|
||
else if (c === ',') { row.push(field); field = ''; }
|
||
else if (c === '\n') { row.push(field); rows.push(row); row = []; field = ''; }
|
||
else if (c !== '\r') field += c;
|
||
}
|
||
if (field.length > 0 || row.length > 0) { row.push(field); rows.push(row); }
|
||
const [header, ...rest] = rows;
|
||
return rest
|
||
.filter((r) => r.length === header.length && !(r.length === 1 && r[0] === ''))
|
||
.map((r) => Object.fromEntries(header.map((h, i) => [h, r[i]])));
|
||
}
|
||
|
||
function haversineKm(a, b) {
|
||
const toRad = (d) => (d * Math.PI) / 180;
|
||
const dLat = toRad(b.latitude_deg - a.latitude_deg);
|
||
const dLon = toRad(b.longitude_deg - a.longitude_deg);
|
||
const x =
|
||
Math.sin(dLat / 2) ** 2 +
|
||
Math.cos(toRad(a.latitude_deg)) * Math.cos(toRad(b.latitude_deg)) * Math.sin(dLon / 2) ** 2;
|
||
return 2 * 6371 * Math.asin(Math.min(1, Math.sqrt(x)));
|
||
}
|
||
|
||
// "Biggest" proxy — the CSV carries no passenger/traffic data, so rank by service signals:
|
||
// scheduled_service first, then an IATA code (commercially served), then a Wikipedia page,
|
||
// then file order. ponytail: good-enough proxy for hub-vs-satellite; swap in pax data if it ever exists.
|
||
function biggestOf(members) {
|
||
const rank = (r) => [r.scheduled_service === 'yes' ? 1 : 0, r.iata_code ? 1 : 0, r.wikipedia_link ? 1 : 0];
|
||
let best = members[0];
|
||
for (const m of members.slice(1)) {
|
||
const rb = rank(m);
|
||
const rk = rank(best);
|
||
if (rb[0] > rk[0] || (rb[0] === rk[0] && rb[1] > rk[1]) || (rb[0] === rk[0] && rb[1] === rk[1] && rb[2] > rk[2])) best = m;
|
||
}
|
||
return best; // ties keep the lowest file order (members are in file order)
|
||
}
|
||
|
||
async function download(url, attempts = 3) {
|
||
for (let i = 1; i <= attempts; i++) {
|
||
try {
|
||
const res = await fetch(url);
|
||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||
const text = await res.text();
|
||
if (!(text.startsWith('"id"') || text.startsWith('id,')) || text.split('\n').length < 10_000) throw new Error('response does not look like the OurAirports CSV');
|
||
return text;
|
||
} catch (err) {
|
||
console.error(`download attempt ${i}/${attempts} failed: ${err.message}`);
|
||
if (i === attempts) throw err;
|
||
await new Promise((r) => setTimeout(r, 2000 * i));
|
||
}
|
||
}
|
||
throw new Error('unreachable');
|
||
}
|
||
|
||
const text = await download(SOURCE_URL);
|
||
const [headerLine] = text.split('\n');
|
||
const header = headerLine.replace(/\r$/, '').split(',').map((h) => h.replace(/^"|"$/g, ''));
|
||
if (header.length !== 19) { console.error(`unexpected header (${header.length} columns): ${headerLine}`); process.exit(1); }
|
||
|
||
const allRows = parseCsv(text);
|
||
const large = allRows.filter((r) => r.type === 'large_airport');
|
||
console.log(`downloaded ${allRows.length} rows; ${large.length} are large_airport`);
|
||
|
||
// Greedy clustering in file order: a row joins the first cluster whose anchor is within the
|
||
// radius, otherwise it starts a new one. Deterministic — same input, same output.
|
||
const clusters = []; // { anchorIdx (into `large`), members }
|
||
for (let i = 0; i < large.length; i++) {
|
||
let joined = false;
|
||
for (const c of clusters) {
|
||
if (haversineKm(large[i], large[c.anchorIdx]) <= CLUSTER_RADIUS_KM) { c.members.push(i); joined = true; break; }
|
||
}
|
||
if (!joined) clusters.push({ anchorIdx: i, members: [i] });
|
||
}
|
||
|
||
const kept = [];
|
||
const multiClusters = [];
|
||
for (const c of clusters) {
|
||
const winner = biggestOf(c.members.map((i) => large[i]));
|
||
kept.push(winner);
|
||
if (c.members.length > 1) {
|
||
multiClusters.push({
|
||
city: winner.municipality || winner.name,
|
||
keptIcao: winner.icao_code || winner.ident,
|
||
dropped: c.members.map((i) => large[i]).filter((r) => r !== winner).map((r) => `${r.icao_code || r.ident} (${r.name})`),
|
||
});
|
||
}
|
||
}
|
||
|
||
// --- Guards (fail loud before writing anything) ---
|
||
const key = (r) => r.icao_code || r.ident; // the effective ICAO consumers key by (toGeoJson fallback)
|
||
const keys = new Set(kept.map(key));
|
||
if (keys.size !== kept.length) { console.error('duplicate ICAO in curated set — aborting'); process.exit(1); }
|
||
for (const must of ['EDDF', 'UNNT', 'LPPD']) { // EDDF is the seed hub; UNNT (Novosibirsk) + LPPD (Ponta Delgada) pin the 409-backstop test pair
|
||
if (!keys.has(must)) { console.error(`required airport ${must} missing from curated set — aborting`); process.exit(1); }
|
||
}
|
||
if (kept.length < 300 || kept.length > 2500) { console.error(`unexpected row count ${kept.length}`); process.exit(1); }
|
||
|
||
await fs.writeFile(OUT_FILE, [headerLine.replace(/\r$/, ''), ...kept.map((r) => header.map((h) => `"${String(r[h] ?? '').replace(/"/g, '""')}"`).join(','))].join('\n') + '\n', 'utf8');
|
||
|
||
console.log(`wrote ${OUT_FILE}: ${kept.length} rows kept, ${large.length - kept.length} dropped into ${multiClusters.length} multi-airport clusters`);
|
||
console.log('sample merged cities (kept ← dropped):');
|
||
for (const c of multiClusters.slice(0, 15)) console.log(` ${c.city}: kept ${c.keptIcao} ← ${c.dropped.join(', ')}`);
|