236 lines
8.2 KiB
TypeScript
236 lines
8.2 KiB
TypeScript
import type { Job } from "bullmq";
|
||
import { execSync, spawn } from "child_process";
|
||
import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "fs";
|
||
import type { JobProgress } from "@transportationer/shared";
|
||
|
||
export type BuildValhallaData = {
|
||
type: "build-valhalla";
|
||
/** City being added/updated. Absent for removal-only rebuilds. */
|
||
citySlug?: string;
|
||
pbfPath?: string;
|
||
bbox?: [number, number, number, number];
|
||
/** Slugs to drop from the global routing tile set before rebuilding */
|
||
removeSlugs?: string[];
|
||
};
|
||
|
||
const OSM_DATA_DIR = process.env.OSM_DATA_DIR ?? "/data/osm";
|
||
const VALHALLA_CONFIG = process.env.VALHALLA_CONFIG ?? "/data/valhalla/valhalla.json";
|
||
const VALHALLA_TILES_DIR = process.env.VALHALLA_TILES_DIR ?? "/data/valhalla/valhalla_tiles";
|
||
const VALHALLA_DATA_DIR = "/data/valhalla";
|
||
|
||
/**
|
||
* Manifest file: maps citySlug → absolute path of its routing PBF.
|
||
* Persists in the valhalla_tiles Docker volume across restarts.
|
||
*
|
||
* For bbox-clipped cities the path is /data/valhalla/{slug}-routing.osm.pbf.
|
||
* For whole-region cities (no bbox) the path is /data/osm/{slug}-latest.osm.pbf
|
||
* (accessible via the osm_data volume mounted read-only in this container).
|
||
*/
|
||
const ROUTING_MANIFEST = `${VALHALLA_DATA_DIR}/routing-sources.json`;
|
||
|
||
function readManifest(): Record<string, string> {
|
||
try {
|
||
return JSON.parse(readFileSync(ROUTING_MANIFEST, "utf8")) as Record<string, string>;
|
||
} catch {
|
||
return {};
|
||
}
|
||
}
|
||
|
||
function writeManifest(manifest: Record<string, string>): void {
|
||
writeFileSync(ROUTING_MANIFEST, JSON.stringify(manifest, null, 2));
|
||
}
|
||
|
||
function runProcess(cmd: string, args: string[]): Promise<void> {
|
||
return new Promise((resolve, reject) => {
|
||
console.log(`[build-valhalla] Running: ${cmd} ${args.join(" ")}`);
|
||
const child = spawn(cmd, args, { stdio: "inherit" });
|
||
child.on("error", reject);
|
||
child.on("exit", (code) => {
|
||
if (code === 0) resolve();
|
||
else reject(new Error(`${cmd} exited with code ${code}`));
|
||
});
|
||
});
|
||
}
|
||
|
||
type JsonObject = Record<string, unknown>;
|
||
|
||
/** Deep-merge override into base. Objects are merged recursively; arrays and
|
||
* scalars in override replace the corresponding base value entirely. */
|
||
function deepMerge(base: JsonObject, override: JsonObject): JsonObject {
|
||
const result: JsonObject = { ...base };
|
||
for (const [key, val] of Object.entries(override)) {
|
||
const baseVal = base[key];
|
||
if (
|
||
val !== null && typeof val === "object" && !Array.isArray(val) &&
|
||
baseVal !== null && typeof baseVal === "object" && !Array.isArray(baseVal)
|
||
) {
|
||
result[key] = deepMerge(baseVal as JsonObject, val as JsonObject);
|
||
} else {
|
||
result[key] = val;
|
||
}
|
||
}
|
||
return result;
|
||
}
|
||
|
||
/**
|
||
* Generate valhalla.json by starting from the canonical defaults produced by
|
||
* valhalla_build_config, then overlaying only the deployment-specific settings.
|
||
* This ensures every required field for the installed Valhalla version is present
|
||
* without us having to maintain a manual list.
|
||
*/
|
||
function generateConfig(): void {
|
||
mkdirSync(VALHALLA_TILES_DIR, { recursive: true });
|
||
|
||
// Get the full default config for this exact Valhalla build.
|
||
let base: JsonObject = {};
|
||
try {
|
||
const out = execSync("valhalla_build_config", {
|
||
encoding: "utf8",
|
||
maxBuffer: 10 * 1024 * 1024,
|
||
});
|
||
base = JSON.parse(out) as JsonObject;
|
||
console.log("[build-valhalla] Loaded defaults from valhalla_build_config");
|
||
} catch (err) {
|
||
console.warn("[build-valhalla] valhalla_build_config failed, using empty base:", err);
|
||
}
|
||
|
||
// Only override settings specific to this deployment.
|
||
const overrides: JsonObject = {
|
||
mjolnir: {
|
||
tile_dir: VALHALLA_TILES_DIR,
|
||
tile_extract: `${VALHALLA_TILES_DIR}.tar`,
|
||
timezone: `${VALHALLA_TILES_DIR}/timezone.sqlite`,
|
||
admin: `${VALHALLA_TILES_DIR}/admins.sqlite`,
|
||
},
|
||
additional_data: {
|
||
elevation: "/data/elevation/",
|
||
},
|
||
httpd: {
|
||
service: {
|
||
listen: "tcp://*:8002",
|
||
timeout_seconds: 26,
|
||
},
|
||
},
|
||
};
|
||
|
||
const config = deepMerge(base, overrides);
|
||
writeFileSync(VALHALLA_CONFIG, JSON.stringify(config, null, 2));
|
||
console.log(`[build-valhalla] Config written to ${VALHALLA_CONFIG}`);
|
||
}
|
||
|
||
export async function handleBuildValhalla(
|
||
job: Job<BuildValhallaData>,
|
||
restartService: () => Promise<void>,
|
||
): Promise<void> {
|
||
const { citySlug, pbfPath, bbox, removeSlugs = [] } = job.data;
|
||
|
||
// Always regenerate config to ensure it's valid JSON (not stale/corrupted).
|
||
await job.updateProgress({
|
||
stage: "Building routing graph",
|
||
pct: 2,
|
||
message: "Writing Valhalla configuration…",
|
||
} satisfies JobProgress);
|
||
generateConfig();
|
||
|
||
// ── Step 1: update the routing manifest ──────────────────────────────────
|
||
// The manifest maps citySlug → pbfPath for every city that should be
|
||
// included in the global tile set. It persists across container restarts.
|
||
|
||
const manifest = readManifest();
|
||
|
||
// Remove requested cities
|
||
for (const slug of removeSlugs) {
|
||
const clippedPbf = `${VALHALLA_DATA_DIR}/${slug}-routing.osm.pbf`;
|
||
if (existsSync(clippedPbf)) {
|
||
unlinkSync(clippedPbf);
|
||
console.log(`[build-valhalla] Removed clipped PBF for ${slug}`);
|
||
}
|
||
delete manifest[slug];
|
||
}
|
||
|
||
// Add/update the city being ingested (absent for removal-only jobs)
|
||
if (citySlug && pbfPath) {
|
||
await job.updateProgress({
|
||
stage: "Building routing graph",
|
||
pct: 5,
|
||
message: bbox
|
||
? `Clipping PBF to bbox [${bbox.map((v) => v.toFixed(3)).join(", ")}]…`
|
||
: `Registering full PBF for ${citySlug}…`,
|
||
} satisfies JobProgress);
|
||
|
||
let routingPbf: string;
|
||
|
||
if (bbox) {
|
||
const [minLng, minLat, maxLng, maxLat] = bbox;
|
||
const clippedPbf = `${VALHALLA_DATA_DIR}/${citySlug}-routing.osm.pbf`;
|
||
|
||
if (!existsSync(pbfPath)) throw new Error(`PBF file not found: ${pbfPath}`);
|
||
|
||
await runProcess("osmium", [
|
||
"extract",
|
||
`--bbox=${minLng},${minLat},${maxLng},${maxLat}`,
|
||
pbfPath,
|
||
"-o", clippedPbf,
|
||
"--overwrite",
|
||
]);
|
||
routingPbf = clippedPbf;
|
||
} else {
|
||
// No bbox: use the full PBF from the osm_data volume (mounted :ro here)
|
||
if (existsSync(pbfPath)) {
|
||
routingPbf = pbfPath;
|
||
} else {
|
||
const { readdirSync } = await import("fs");
|
||
const found = readdirSync(OSM_DATA_DIR)
|
||
.filter((f) => f.endsWith("-latest.osm.pbf"))
|
||
.map((f) => `${OSM_DATA_DIR}/${f}`);
|
||
if (found.length === 0) throw new Error(`No PBF files found in ${OSM_DATA_DIR}`);
|
||
routingPbf = found[0];
|
||
}
|
||
}
|
||
|
||
manifest[citySlug] = routingPbf;
|
||
}
|
||
|
||
writeManifest(manifest);
|
||
|
||
// ── Step 2: build tiles from ALL registered cities ────────────────────────
|
||
|
||
const allPbfs = Object.values(manifest).filter(existsSync);
|
||
const allSlugs = Object.keys(manifest);
|
||
|
||
if (allPbfs.length === 0) {
|
||
console.log("[build-valhalla] Manifest is empty — no cities to build routing tiles for.");
|
||
await job.updateProgress({
|
||
stage: "Building routing graph",
|
||
pct: 100,
|
||
message: "No cities in manifest, skipping tile build.",
|
||
} satisfies JobProgress);
|
||
return;
|
||
}
|
||
|
||
await job.updateProgress({
|
||
stage: "Building routing graph",
|
||
pct: 10,
|
||
message: `Building global routing tiles for: ${allSlugs.join(", ")}`,
|
||
} satisfies JobProgress);
|
||
|
||
// valhalla_build_tiles accepts multiple PBF files as positional arguments,
|
||
// so we get one combined tile set covering all cities in a single pass.
|
||
await runProcess("valhalla_build_tiles", ["-c", VALHALLA_CONFIG, ...allPbfs]);
|
||
|
||
// Tiles are fully built — restart the service to pick them up.
|
||
// compute-routing jobs will transparently retry their in-flight matrix calls
|
||
// across the brief restart window (~5–10 s).
|
||
await job.updateProgress({
|
||
stage: "Building routing graph",
|
||
pct: 95,
|
||
message: "Tiles built — restarting Valhalla service…",
|
||
} satisfies JobProgress);
|
||
await restartService();
|
||
|
||
await job.updateProgress({
|
||
stage: "Building routing graph",
|
||
pct: 100,
|
||
message: `Routing graph ready — covers: ${allSlugs.join(", ")}`,
|
||
} satisfies JobProgress);
|
||
}
|