import type { Job } from "bullmq"; import { execSync, spawn } from "child_process"; import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"; import type { JobProgress } from "@transportationer/shared"; export type BuildValhallaData = { type: "build-valhalla"; /** City being added/updated. Absent for removal-only rebuilds. */ citySlug?: string; pbfPath?: string; bbox?: [number, number, number, number]; /** Slugs to drop from the global routing tile set before rebuilding */ removeSlugs?: string[]; }; const OSM_DATA_DIR = process.env.OSM_DATA_DIR ?? "/data/osm"; const VALHALLA_CONFIG = process.env.VALHALLA_CONFIG ?? "/data/valhalla/valhalla.json"; const VALHALLA_TILES_DIR = process.env.VALHALLA_TILES_DIR ?? "/data/valhalla/valhalla_tiles"; const VALHALLA_DATA_DIR = "/data/valhalla"; /** * Manifest file: maps citySlug → absolute path of its routing PBF. * Persists in the valhalla_tiles Docker volume across restarts. * * For bbox-clipped cities the path is /data/valhalla/{slug}-routing.osm.pbf. * For whole-region cities (no bbox) the path is /data/osm/{slug}-latest.osm.pbf * (accessible via the osm_data volume mounted read-only in this container). */ const ROUTING_MANIFEST = `${VALHALLA_DATA_DIR}/routing-sources.json`; function readManifest(): Record { try { return JSON.parse(readFileSync(ROUTING_MANIFEST, "utf8")) as Record; } catch { return {}; } } function writeManifest(manifest: Record): void { writeFileSync(ROUTING_MANIFEST, JSON.stringify(manifest, null, 2)); } function runProcess(cmd: string, args: string[]): Promise { return new Promise((resolve, reject) => { console.log(`[build-valhalla] Running: ${cmd} ${args.join(" ")}`); const child = spawn(cmd, args, { stdio: "inherit" }); child.on("error", reject); child.on("exit", (code) => { if (code === 0) resolve(); else reject(new Error(`${cmd} exited with code ${code}`)); }); }); } type JsonObject = Record; /** Deep-merge override into base. Objects are merged recursively; arrays and * scalars in override replace the corresponding base value entirely. */ function deepMerge(base: JsonObject, override: JsonObject): JsonObject { const result: JsonObject = { ...base }; for (const [key, val] of Object.entries(override)) { const baseVal = base[key]; if ( val !== null && typeof val === "object" && !Array.isArray(val) && baseVal !== null && typeof baseVal === "object" && !Array.isArray(baseVal) ) { result[key] = deepMerge(baseVal as JsonObject, val as JsonObject); } else { result[key] = val; } } return result; } /** * Generate valhalla.json by starting from the canonical defaults produced by * valhalla_build_config, then overlaying only the deployment-specific settings. * This ensures every required field for the installed Valhalla version is present * without us having to maintain a manual list. */ function generateConfig(): void { mkdirSync(VALHALLA_TILES_DIR, { recursive: true }); // Get the full default config for this exact Valhalla build. let base: JsonObject = {}; try { const out = execSync("valhalla_build_config", { encoding: "utf8", maxBuffer: 10 * 1024 * 1024, }); base = JSON.parse(out) as JsonObject; console.log("[build-valhalla] Loaded defaults from valhalla_build_config"); } catch (err) { console.warn("[build-valhalla] valhalla_build_config failed, using empty base:", err); } // Only override settings specific to this deployment. const overrides: JsonObject = { mjolnir: { tile_dir: VALHALLA_TILES_DIR, tile_extract: `${VALHALLA_TILES_DIR}.tar`, timezone: `${VALHALLA_TILES_DIR}/timezone.sqlite`, admin: `${VALHALLA_TILES_DIR}/admins.sqlite`, }, additional_data: { elevation: "/data/elevation/", }, httpd: { service: { listen: "tcp://*:8002", timeout_seconds: 26, }, }, }; const config = deepMerge(base, overrides); writeFileSync(VALHALLA_CONFIG, JSON.stringify(config, null, 2)); console.log(`[build-valhalla] Config written to ${VALHALLA_CONFIG}`); } export async function handleBuildValhalla( job: Job, restartService: () => Promise, ): Promise { const { citySlug, pbfPath, bbox, removeSlugs = [] } = job.data; // Always regenerate config to ensure it's valid JSON (not stale/corrupted). await job.updateProgress({ stage: "Building routing graph", pct: 2, message: "Writing Valhalla configuration…", } satisfies JobProgress); generateConfig(); // ── Step 1: update the routing manifest ────────────────────────────────── // The manifest maps citySlug → pbfPath for every city that should be // included in the global tile set. It persists across container restarts. const manifest = readManifest(); // Remove requested cities for (const slug of removeSlugs) { const clippedPbf = `${VALHALLA_DATA_DIR}/${slug}-routing.osm.pbf`; if (existsSync(clippedPbf)) { unlinkSync(clippedPbf); console.log(`[build-valhalla] Removed clipped PBF for ${slug}`); } delete manifest[slug]; } // Add/update the city being ingested (absent for removal-only jobs) if (citySlug && pbfPath) { await job.updateProgress({ stage: "Building routing graph", pct: 5, message: bbox ? `Clipping PBF to bbox [${bbox.map((v) => v.toFixed(3)).join(", ")}]…` : `Registering full PBF for ${citySlug}…`, } satisfies JobProgress); let routingPbf: string; if (bbox) { const [minLng, minLat, maxLng, maxLat] = bbox; const clippedPbf = `${VALHALLA_DATA_DIR}/${citySlug}-routing.osm.pbf`; if (!existsSync(pbfPath)) throw new Error(`PBF file not found: ${pbfPath}`); await runProcess("osmium", [ "extract", `--bbox=${minLng},${minLat},${maxLng},${maxLat}`, pbfPath, "-o", clippedPbf, "--overwrite", ]); routingPbf = clippedPbf; } else { // No bbox: use the full PBF from the osm_data volume (mounted :ro here) if (existsSync(pbfPath)) { routingPbf = pbfPath; } else { const { readdirSync } = await import("fs"); const found = readdirSync(OSM_DATA_DIR) .filter((f) => f.endsWith("-latest.osm.pbf")) .map((f) => `${OSM_DATA_DIR}/${f}`); if (found.length === 0) throw new Error(`No PBF files found in ${OSM_DATA_DIR}`); routingPbf = found[0]; } } manifest[citySlug] = routingPbf; } writeManifest(manifest); // ── Step 2: build tiles from ALL registered cities ──────────────────────── const allPbfs = Object.values(manifest).filter(existsSync); const allSlugs = Object.keys(manifest); if (allPbfs.length === 0) { console.log("[build-valhalla] Manifest is empty — no cities to build routing tiles for."); await job.updateProgress({ stage: "Building routing graph", pct: 100, message: "No cities in manifest, skipping tile build.", } satisfies JobProgress); return; } await job.updateProgress({ stage: "Building routing graph", pct: 10, message: `Building global routing tiles for: ${allSlugs.join(", ")}`, } satisfies JobProgress); // valhalla_build_tiles accepts multiple PBF files as positional arguments, // so we get one combined tile set covering all cities in a single pass. await runProcess("valhalla_build_tiles", ["-c", VALHALLA_CONFIG, ...allPbfs]); // Tiles are fully built — restart the service to pick them up. // compute-routing jobs will transparently retry their in-flight matrix calls // across the brief restart window (~5–10 s). await job.updateProgress({ stage: "Building routing graph", pct: 95, message: "Tiles built — restarting Valhalla service…", } satisfies JobProgress); await restartService(); await job.updateProgress({ stage: "Building routing graph", pct: 100, message: `Routing graph ready — covers: ${allSlugs.join(", ")}`, } satisfies JobProgress); }