diff --git a/worker/src/jobs/build-valhalla.ts b/worker/src/jobs/build-valhalla.ts index 2005662..581e23d 100644 --- a/worker/src/jobs/build-valhalla.ts +++ b/worker/src/jobs/build-valhalla.ts @@ -1,3 +1,4 @@ +import { cpus } from "os"; import type { Job } from "bullmq"; import { execSync, spawn } from "child_process"; import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, unlinkSync, writeFileSync } from "fs"; @@ -61,6 +62,13 @@ const INCLUDE_TRANSIT = (process.env.VALHALLA_INCLUDE_TRANSIT ?? "").toLowerCase const ROAD_BBOX_BUFFER = 0.2; +/** + * Cap Valhalla worker thread count. Defaults to CPU count which can be 256+ + * on large hosts — causes OOM for cities with large transit datasets (Berlin etc.). + * Override with VALHALLA_CONCURRENCY env var. + */ +const VALHALLA_CONCURRENCY = parseInt(process.env.VALHALLA_CONCURRENCY ?? String(Math.max(1, Math.floor(cpus().length / 2))), 10); + const ROUTING_MANIFEST = `${VALHALLA_DATA_DIR}/routing-sources.json`; function readManifest(): Record { @@ -80,9 +88,9 @@ function runProcess(cmd: string, args: string[]): Promise { console.log(`[build-valhalla] Running: ${cmd} ${args.join(" ")}`); const child = spawn(cmd, args, { stdio: "inherit" }); child.on("error", reject); - child.on("exit", (code) => { + child.on("exit", (code, signal) => { if (code === 0) resolve(); - else reject(new Error(`${cmd} exited with code ${code}`)); + else reject(new Error(`${cmd} exited with code ${code}${signal ? ` (signal: ${signal})` : ""}`)); }); }); } @@ -148,6 +156,7 @@ function writeCityTransitIngestConfig(citySlug: string): string { admin: ADMINS_SQLITE, transit_dir: cityTransitCacheDir(citySlug), transit_feeds_dir: cityGtfsFeedsParentDir(citySlug), + concurrency: VALHALLA_CONCURRENCY, }, service_limits: { isochrone: { max_contours: 5 } }, } as JsonObject); @@ -170,6 +179,7 @@ function writeCityConfig(citySlug: string, cityTileDir: string): string { tile_extract: `${cityTileDir}.tar`, timezone: TIMEZONE_SQLITE, admin: ADMINS_SQLITE, + concurrency: VALHALLA_CONCURRENCY, ...(INCLUDE_TRANSIT ? { transit_dir: cityTransitCacheDir(citySlug), transit_feeds_dir: cityGtfsFeedsParentDir(citySlug),