jan-henrik/fifteen
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: add selection by city boundary, show that, stats only for city, not bbox

Browse source
This commit is contained in:
Jan-Henrik 2026-03-05 00:38:21 +01:00
parent b891ca79ac
commit e94d660686
12 changed files with 482 additions and 302 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

411
apps/web/app/admin/cities/new/page.tsx
View file

@ -3,6 +3,7 @@
import { useState, useEffect, useMemo, useRef, useCallback } from "react"; import { useState, useEffect, useMemo, useRef, useCallback } from "react";
import type { GeofabrikFeature, GeofabrikIndex } from "@transportationer/shared"; import type { GeofabrikFeature, GeofabrikIndex } from "@transportationer/shared";
import { useJobProgress } from "@/hooks/use-job-progress"; import { useJobProgress } from "@/hooks/use-job-progress";
import type { StageStatus, RoutingDetail as RoutingDetailType } from "@/hooks/use-job-progress";
type Step = "browse" | "confirm" | "ingest"; type Step = "browse" | "confirm" | "ingest";
@ -186,7 +187,7 @@ function GeofabrikBrowser({
); );
} }
// ─── Nominatim geocoder + radius selector + mini map ───────────────────────── // ─── Nominatim polygon geocoder + mini map ───────────────────────────────────
interface NominatimResult { interface NominatimResult {
place_id: number; place_id: number;
@ -195,71 +196,34 @@ interface NominatimResult {
lon: string; lon: string;
type: string; type: string;
class: string; class: string;
} geojson?: { type: string; coordinates: unknown };
const RADIUS_OPTIONS = [2, 5, 10, 15, 20, 30];
function computeBbox(
lat: number,
lng: number,
radiusKm: number,
): [number, number, number, number] {
const latDelta = radiusKm / 111.0;
const lngDelta = radiusKm / (111.0 * Math.cos((lat * Math.PI) / 180));
return [lng - lngDelta, lat - latDelta, lng + lngDelta, lat + latDelta];
}
function bboxToGeoJSON(bbox: [number, number, number, number]) {
const [w, s, e, n] = bbox;
return {
type: "Feature" as const,
geometry: {
type: "Polygon" as const,
coordinates: [
[
[w, s],
[e, s],
[e, n],
[w, n],
[w, s],
],
],
},
properties: {},
};
} }
function LocationSelector({ function LocationSelector({
regionGeometry, regionGeometry,
onBboxChange, onBoundaryChange,
}: { }: {
regionGeometry: GeofabrikFeature["geometry"]; regionGeometry: GeofabrikFeature["geometry"];
onBboxChange: (bbox: [number, number, number, number] | null) => void; onBoundaryChange: (boundary: { type: string; coordinates: unknown } | null) => void;
}) { }) {
const [query, setQuery] = useState(""); const [query, setQuery] = useState("");
const [results, setResults] = useState<NominatimResult[]>([]); const [results, setResults] = useState<NominatimResult[]>([]);
const [selected, setSelected] = useState<NominatimResult | null>(null); const [selected, setSelected] = useState<NominatimResult | null>(null);
const [radius, setRadius] = useState(10);
const [showDropdown, setShowDropdown] = useState(false); const [showDropdown, setShowDropdown] = useState(false);
const [searching, setSearching] = useState(false); const [searching, setSearching] = useState(false);
const mapContainerRef = useRef<HTMLDivElement>(null); const mapContainerRef = useRef<HTMLDivElement>(null);
const mapRef = useRef<import("maplibre-gl").Map | null>(null); const mapRef = useRef<import("maplibre-gl").Map | null>(null);
const mapReadyRef = useRef(false); const mapReadyRef = useRef(false);
const onBboxChangeRef = useRef(onBboxChange); const onBoundaryChangeRef = useRef(onBoundaryChange);
onBboxChangeRef.current = onBboxChange; onBoundaryChangeRef.current = onBoundaryChange;
const bbox = useMemo((): [number, number, number, number] | null => { // Notify parent when selection changes
if (!selected) return null;
return computeBbox(parseFloat(selected.lat), parseFloat(selected.lon), radius);
}, [selected, radius]);
// Notify parent when bbox changes
useEffect(() => { useEffect(() => {
onBboxChangeRef.current(bbox); onBoundaryChangeRef.current(selected?.geojson ?? null);
}, [bbox]); }, [selected]);
// Debounced Nominatim search // Debounced Nominatim search — request polygon_geojson + featuretype=settlement
useEffect(() => { useEffect(() => {
if (query.length < 2) { if (query.length < 2) {
setResults([]); setResults([]);
@ -269,11 +233,15 @@ function LocationSelector({
setSearching(true); setSearching(true);
try { try {
const res = await fetch( const res = await fetch(
`https://nominatim.openstreetmap.org/search?q=${encodeURIComponent(query)}&format=json&limit=6&addressdetails=0`, `https://nominatim.openstreetmap.org/search?q=${encodeURIComponent(query)}&polygon_geojson=1&featuretype=settlement&format=json&limit=8`,
{ headers: { "User-Agent": "Transportationer/1.0 (15-minute city analyzer)" } }, { headers: { "User-Agent": "Transportationer/1.0 (15-minute city analyzer)" } },
); );
const data: NominatimResult[] = await res.json(); const data: NominatimResult[] = await res.json();
setResults(data); // Keep only results that have a real polygon boundary
const polygons = data.filter(
(r) => r.geojson?.type === "Polygon" || r.geojson?.type === "MultiPolygon",
);
setResults(polygons);
setShowDropdown(true); setShowDropdown(true);
} catch { } catch {
setResults([]); setResults([]);
@ -284,15 +252,27 @@ function LocationSelector({
return () => clearTimeout(timer); return () => clearTimeout(timer);
}, [query]); }, [query]);
// Apply bbox to mini map // Apply boundary polygon to mini map
const applyBboxToMap = useCallback( const applyBoundaryToMap = useCallback(
(map: import("maplibre-gl").Map, b: [number, number, number, number] | null) => { (map: import("maplibre-gl").Map, geojson: NominatimResult["geojson"] | null) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
const src = map.getSource("area-bbox") as any; const src = map.getSource("area-boundary") as any;
if (!src) return; if (!src) return;
if (b) { if (geojson) {
src.setData(bboxToGeoJSON(b)); src.setData({ type: "Feature", geometry: geojson, properties: {} });
map.fitBounds([b[0], b[1], b[2], b[3]], { padding: 40, duration: 500 }); // Fit to boundary bbox
try {
const coords: number[][] =
geojson.type === "Polygon"
? (geojson.coordinates as number[][][])[0]
: (geojson.coordinates as number[][][][])[0][0];
const lngs = coords.map((c) => c[0]);
const lats = coords.map((c) => c[1]);
map.fitBounds(
[Math.min(...lngs), Math.min(...lats), Math.max(...lngs), Math.max(...lats)],
{ padding: 40, duration: 500 },
);
} catch { /* ignore */ }
} else { } else {
src.setData({ type: "FeatureCollection", features: [] }); src.setData({ type: "FeatureCollection", features: [] });
} }
@ -300,11 +280,11 @@ function LocationSelector({
[], [],
); );
// Update mini map when bbox changes // Update mini map when selection changes
useEffect(() => { useEffect(() => {
const map = mapRef.current; const map = mapRef.current;
if (map && mapReadyRef.current) applyBboxToMap(map, bbox); if (map && mapReadyRef.current) applyBoundaryToMap(map, selected?.geojson ?? null);
}, [bbox, applyBboxToMap]); }, [selected, applyBoundaryToMap]);
// Initialize mini map // Initialize mini map
useEffect(() => { useEffect(() => {
@ -357,49 +337,39 @@ function LocationSelector({
paint: { "line-color": "#64748b", "line-width": 1.5, "line-dasharray": [3, 2] }, paint: { "line-color": "#64748b", "line-width": 1.5, "line-dasharray": [3, 2] },
}); });
// Selected sub-area bbox // Selected city boundary polygon
map.addSource("area-bbox", { map.addSource("area-boundary", {
type: "geojson", type: "geojson",
data: { type: "FeatureCollection", features: [] }, data: { type: "FeatureCollection", features: [] },
}); });
map.addLayer({ map.addLayer({
id: "area-bbox-fill", id: "area-boundary-fill",
type: "fill", type: "fill",
source: "area-bbox", source: "area-boundary",
paint: { "fill-color": "#2563eb", "fill-opacity": 0.15 }, paint: { "fill-color": "#2563eb", "fill-opacity": 0.15 },
}); });
map.addLayer({ map.addLayer({
id: "area-bbox-line", id: "area-boundary-line",
type: "line", type: "line",
source: "area-bbox", source: "area-boundary",
paint: { "line-color": "#2563eb", "line-width": 2 }, paint: { "line-color": "#2563eb", "line-width": 2 },
}); });
// Fit to region if available // Fit to region if available
if (regionGeometry) { if (regionGeometry) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any try {
const src = map.getSource("area-region") as any; const coords: number[][] =
if (src) { regionGeometry.type === "Polygon"
// Fit map to the region outline ? regionGeometry.coordinates[0]
try { : regionGeometry.coordinates[0][0];
const coords: number[][] = const lngs = coords.map((c) => c[0]);
regionGeometry.type === "Polygon" const lats = coords.map((c) => c[1]);
? regionGeometry.coordinates[0] map.fitBounds(
: regionGeometry.coordinates[0][0]; [Math.min(...lngs), Math.min(...lats), Math.max(...lngs), Math.max(...lats)],
const lngs = coords.map((c) => c[0]); { padding: 20 },
const lats = coords.map((c) => c[1]); );
map.fitBounds( } catch { /* ignore */ }
[Math.min(...lngs), Math.min(...lats), Math.max(...lngs), Math.max(...lats)],
{ padding: 20 },
);
} catch {
/* ignore */
}
}
} }
// Apply bbox if already set (e.g. after component re-render)
if (bbox) applyBboxToMap(map, bbox);
}); });
})(); })();
@ -418,7 +388,7 @@ function LocationSelector({
<div className="flex gap-2 items-center"> <div className="flex gap-2 items-center">
<input <input
type="search" type="search"
placeholder="Search for a city or location…" placeholder="Search for a city or municipality…"
value={query} value={query}
onChange={(e) => { onChange={(e) => {
setQuery(e.target.value); setQuery(e.target.value);
@ -451,35 +421,13 @@ function LocationSelector({
<span className="text-gray-400 ml-1 text-xs"> <span className="text-gray-400 ml-1 text-xs">
{r.display_name.split(",").slice(1, 3).join(",")} {r.display_name.split(",").slice(1, 3).join(",")}
</span> </span>
<span className="ml-2 text-xs text-blue-500">{r.geojson?.type}</span>
</button> </button>
))} ))}
</div> </div>
)} )}
</div> </div>
{/* Radius selector */}
{selected && (
<div className="flex items-center gap-3">
<label className="text-sm text-gray-600 shrink-0">Radius:</label>
<div className="flex gap-1.5 flex-wrap">
{RADIUS_OPTIONS.map((r) => (
<button
key={r}
type="button"
onClick={() => setRadius(r)}
className={`px-2.5 py-1 rounded-full text-xs font-medium border transition-colors ${
radius === r
? "bg-brand-600 text-white border-brand-600"
: "bg-white text-gray-600 border-gray-300 hover:border-brand-400"
}`}
>
{r} km
</button>
))}
</div>
</div>
)}
{/* Mini map */} {/* Mini map */}
<div <div
ref={mapContainerRef} ref={mapContainerRef}
@ -487,14 +435,14 @@ function LocationSelector({
style={{ height: 220 }} style={{ height: 220 }}
/> />
{bbox && ( {selected?.geojson && (
<p className="text-xs text-green-700"> <p className="text-xs text-green-700">
✓ Sub-region: {radius} km around {selected!.display_name.split(",")[0]} — bbox [{bbox.map((v) => v.toFixed(4)).join(", ")}] ✓ Boundary: {selected.display_name.split(",").slice(0, 2).join(", ")} ({selected.geojson.type})
</p> </p>
)} )}
{!selected && ( {!selected && (
<p className="text-xs text-gray-400"> <p className="text-xs text-gray-400">
Search for a location to select a sub-region, or leave empty to use the entire dataset. Search for a city to use its administrative boundary for stats coverage, or leave empty to use the entire dataset.
</p> </p>
)} )}
</div> </div>
@ -514,18 +462,18 @@ function ConfirmStep({
slug: string, slug: string,
name: string, name: string,
countryCode: string, countryCode: string,
bbox: [number, number, number, number] | null, boundary: { type: string; coordinates: unknown } | null,
) => Promise<void>; ) => Promise<void>;
}) { }) {
const defaultSlug = region.properties.id.replace(/\//g, "-"); const defaultSlug = region.properties.id.replace(/\//g, "-");
const [slug, setSlug] = useState(defaultSlug); const [slug, setSlug] = useState(defaultSlug);
const [name, setName] = useState(region.properties.name); const [name, setName] = useState(region.properties.name);
const [countryCode, setCountryCode] = useState(""); const [countryCode, setCountryCode] = useState("");
const [bbox, setBbox] = useState<[number, number, number, number] | null>(null); const [boundary, setBoundary] = useState<{ type: string; coordinates: unknown } | null>(null);
const [loading, setLoading] = useState(false); const [loading, setLoading] = useState(false);
const handleBboxChange = useCallback( const handleBoundaryChange = useCallback(
(b: [number, number, number, number] | null) => setBbox(b), (b: { type: string; coordinates: unknown } | null) => setBoundary(b),
[], [],
); );
@ -572,20 +520,20 @@ function ConfirmStep({
/> />
</div> </div>
{/* Sub-region selector */} {/* City boundary selector */}
<div> <div>
<label className="block text-sm font-medium text-gray-700 mb-2"> <label className="block text-sm font-medium text-gray-700 mb-2">
Sub-region{" "} City Boundary{" "}
<span className="text-gray-400 font-normal">(optional — clip to a city area)</span> <span className="text-gray-400 font-normal">(optional — for accurate coverage stats)</span>
</label> </label>
<p className="text-xs text-gray-500 mb-3"> <p className="text-xs text-gray-500 mb-3">
Select a city center and radius to ingest only that sub-area of the dataset. Search for the city municipality to use its administrative boundary for
Recommended for large regions (e.g. entire states). Affects OSM import, coverage statistics. The bounding box is still used for OSM import, routing,
routing tiles, and grid analysis. and grid generation.
</p> </p>
<LocationSelector <LocationSelector
regionGeometry={region.geometry} regionGeometry={region.geometry}
onBboxChange={handleBboxChange} onBoundaryChange={handleBoundaryChange}
/> />
</div> </div>
@ -604,7 +552,7 @@ function ConfirmStep({
<button <button
onClick={async () => { onClick={async () => {
setLoading(true); setLoading(true);
await onConfirm(slug, name, countryCode, bbox); await onConfirm(slug, name, countryCode, boundary);
setLoading(false); setLoading(false);
}} }}
disabled={loading || !slug || !name} disabled={loading || !slug || !name}
@ -619,39 +567,147 @@ function ConfirmStep({
// ─── Progress step ──────────────────────────────────────────────────────────── // ─── Progress step ────────────────────────────────────────────────────────────
function StageIcon({ status }: { status: StageStatus["status"] }) {
if (status === "completed")
return (
<span className="w-5 h-5 flex items-center justify-center rounded-full bg-green-100 text-green-600 text-xs shrink-0">
✓
</span>
);
if (status === "failed")
return (
<span className="w-5 h-5 flex items-center justify-center rounded-full bg-red-100 text-red-600 text-xs shrink-0">
✗
</span>
);
if (status === "active")
return (
<span className="w-5 h-5 flex items-center justify-center shrink-0">
<svg className="animate-spin w-4 h-4 text-brand-600" viewBox="0 0 24 24" fill="none">
<circle cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="2" opacity="0.25" />
<path d="M12 2a10 10 0 0 1 10 10" stroke="currentColor" strokeWidth="2" strokeLinecap="round" />
</svg>
</span>
);
return <span className="w-5 h-5 rounded-full border-2 border-gray-300 shrink-0" />;
}
function StageRow({
stage,
error,
}: {
stage: StageStatus;
error?: string;
}) {
return (
<div className="flex items-start gap-3">
<StageIcon status={stage.status} />
<div className="flex-1 min-w-0">
<p className={`text-sm font-medium ${stage.status === "pending" ? "text-gray-400" : "text-gray-900"}`}>
{stage.label}
</p>
{stage.status === "active" && (
<>
<div className="w-full bg-gray-200 rounded-full h-1 mt-1.5">
<div
className="bg-brand-600 h-1 rounded-full transition-all duration-500"
style={{ width: `${stage.pct}%` }}
/>
</div>
<p className="text-xs text-gray-500 mt-0.5 truncate">{stage.message}</p>
</>
)}
{stage.status === "failed" && error && (
<p className="text-xs text-red-600 mt-0.5">{error}</p>
)}
</div>
</div>
);
}
function RoutingGrid({ routingDetail }: { routingDetail: RoutingDetailType }) {
const MODE_LABELS: Record<string, string> = {
walking: "Walking",
cycling: "Cycling",
driving: "Driving",
transit: "Transit",
};
const entries = Object.entries(routingDetail);
if (entries.length === 0) return null;
return (
<div className="mt-2 space-y-1.5 pl-8">
{entries.map(([mode, { done, total }]) => (
<div key={mode} className="flex items-center gap-2">
<span className="text-xs text-gray-500 w-14 shrink-0">{MODE_LABELS[mode] ?? mode}</span>
<div className="flex-1 bg-gray-200 rounded-full h-1.5">
<div
className="bg-brand-500 h-1.5 rounded-full transition-all duration-500"
style={{ width: total > 0 ? `${(done / total) * 100}%` : done > 0 ? "100%" : "0%" }}
/>
</div>
<span className="text-xs text-gray-400 w-10 text-right shrink-0">
{total > 1 ? `${done}/${total}` : done >= 1 ? "done" : "…"}
</span>
</div>
))}
</div>
);
}
function ProgressStep({ jobId }: { jobId: string | null }) { function ProgressStep({ jobId }: { jobId: string | null }) {
const { stages, overall, error } = useJobProgress(jobId); const { stages, overall, error, routingDetail } = useJobProgress(jobId);
// Group consecutive parallel stages together for rendering
type StageGroup =
| { kind: "single"; stage: StageStatus }
| { kind: "parallel"; stages: StageStatus[] };
const groups: StageGroup[] = [];
for (const stage of stages) {
if (stage.parallelGroup) {
const last = groups[groups.length - 1];
if (last?.kind === "parallel" && last.stages[0].parallelGroup === stage.parallelGroup) {
last.stages.push(stage);
} else {
groups.push({ kind: "parallel", stages: [stage] });
}
} else {
groups.push({ kind: "single", stage });
}
}
return ( return (
<div className="card max-w-lg"> <div className="card max-w-lg">
<h2 className="text-lg font-semibold mb-6">Processing City Data</h2> <h2 className="text-lg font-semibold mb-6">Processing City Data</h2>
<ol className="space-y-4"> <div className="space-y-4">
{stages.map((stage) => ( {groups.map((group, gi) =>
<li key={stage.key} className="flex items-start gap-3"> group.kind === "single" ? (
<StageIcon status={stage.status} /> <div key={group.stage.key}>
<div className="flex-1 min-w-0"> <StageRow stage={group.stage} error={error} />
<p className="text-sm font-medium text-gray-900">{stage.label}</p> {/* Show per-mode routing grid under the compute-accessibility stage */}
{stage.status === "active" && ( {group.stage.key === "Computing scores" &&
<> group.stage.status === "active" &&
<div className="w-full bg-gray-200 rounded-full h-1.5 mt-1.5"> routingDetail && (
<div <RoutingGrid routingDetail={routingDetail} />
className="bg-brand-600 h-1.5 rounded-full transition-all duration-500" )}
style={{ width: `${stage.pct}%` }}
/>
</div>
<p className="text-xs text-gray-500 mt-1 truncate">
{stage.message}
</p>
</>
)}
{stage.status === "failed" && error && (
<p className="text-xs text-red-600 mt-1">{error}</p>
)}
</div> </div>
</li> ) : (
))} <div
</ol> key={`group-${gi}`}
className="rounded-lg border border-gray-200 bg-gray-50 p-3 space-y-3"
>
<p className="text-xs font-medium text-gray-400 uppercase tracking-wide">
Running in parallel
</p>
{group.stages.map((s) => (
<StageRow key={s.key} stage={s} error={error} />
))}
</div>
),
)}
</div>
{overall === "completed" && ( {overall === "completed" && (
<div className="mt-6 p-4 bg-green-50 rounded-lg text-green-800 text-sm"> <div className="mt-6 p-4 bg-green-50 rounded-lg text-green-800 text-sm">
@ -669,7 +725,7 @@ function ProgressStep({ jobId }: { jobId: string | null }) {
{overall === "failed" && ( {overall === "failed" && (
<div className="mt-6 p-4 bg-red-50 rounded-lg text-red-800 text-sm"> <div className="mt-6 p-4 bg-red-50 rounded-lg text-red-800 text-sm">
✗ Ingestion failed: {error}.{" "} ✗ Ingestion failed: {error}{" "}
<a href="/admin" className="underline"> <a href="/admin" className="underline">
Return to dashboard Return to dashboard
</a> </a>
@ -680,51 +736,6 @@ function ProgressStep({ jobId }: { jobId: string | null }) {
); );
} }
function StageIcon({ status }: { status: StageStatus["status"] }) {
if (status === "completed")
return (
<span className="w-5 h-5 flex items-center justify-center rounded-full bg-green-100 text-green-600 text-xs mt-0.5 shrink-0">
✓
</span>
);
if (status === "failed")
return (
<span className="w-5 h-5 flex items-center justify-center rounded-full bg-red-100 text-red-600 text-xs mt-0.5 shrink-0">
✗
</span>
);
if (status === "active")
return (
<span className="w-5 h-5 flex items-center justify-center mt-0.5 shrink-0">
<svg
className="animate-spin w-4 h-4 text-brand-600"
viewBox="0 0 24 24"
fill="none"
>
<circle
cx="12"
cy="12"
r="10"
stroke="currentColor"
strokeWidth="2"
opacity="0.25"
/>
<path
d="M12 2a10 10 0 0 1 10 10"
stroke="currentColor"
strokeWidth="2"
strokeLinecap="round"
/>
</svg>
</span>
);
return (
<span className="w-5 h-5 rounded-full border-2 border-gray-300 mt-0.5 shrink-0" />
);
}
import type { StageStatus } from "@/hooks/use-job-progress";
// ─── Main page ──────────────────────────────────────────────────────────────── // ─── Main page ────────────────────────────────────────────────────────────────
export default function AddCityPage() { export default function AddCityPage() {
@ -737,7 +748,7 @@ export default function AddCityPage() {
slug: string, slug: string,
name: string, name: string,
countryCode: string, countryCode: string,
bbox: [number, number, number, number] | null, boundary: { type: string; coordinates: unknown } | null,
) => { ) => {
setIngestError(null); setIngestError(null);
try { try {
@ -749,7 +760,7 @@ export default function AddCityPage() {
name, name,
countryCode, countryCode,
geofabrikUrl: selected!.properties.urls.pbf, geofabrikUrl: selected!.properties.urls.pbf,
...(bbox ? { bbox } : {}), ...(boundary ? { boundary } : {}),
}), }),
}); });
if (!res.ok) { if (!res.ok) {

37
apps/web/app/api/admin/cities/route.ts
View file

@ -51,7 +51,7 @@ export async function POST(req: NextRequest) {
return NextResponse.json({ error: "Invalid JSON" }, { status: 400 }); return NextResponse.json({ error: "Invalid JSON" }, { status: 400 });
} }
const { name, slug, countryCode, geofabrikUrl, resolutionM = 200, bbox } = body; const { name, slug, countryCode, geofabrikUrl, resolutionM = 200, bbox, boundary } = body;
if (typeof slug !== "string" || !/^[a-z0-9-]+$/.test(slug)) { if (typeof slug !== "string" || !/^[a-z0-9-]+$/.test(slug)) {
return NextResponse.json( return NextResponse.json(
@ -67,9 +67,20 @@ export async function POST(req: NextRequest) {
); );
} }
// Validate optional boundary GeoJSON (Polygon or MultiPolygon)
if (boundary !== undefined) {
const b = boundary as Record<string, unknown>;
if (b.type !== "Polygon" && b.type !== "MultiPolygon") {
return NextResponse.json(
{ error: "boundary must be a GeoJSON Polygon or MultiPolygon" },
{ status: 400 },
);
}
}
// Validate optional bbox [minLng, minLat, maxLng, maxLat] // Validate optional bbox [minLng, minLat, maxLng, maxLat]
let validBbox: [number, number, number, number] | null = null; let validBbox: [number, number, number, number] | null = null;
if (bbox !== undefined) { if (bbox !== undefined && boundary === undefined) {
if ( if (
!Array.isArray(bbox) || !Array.isArray(bbox) ||
bbox.length !== 4 || bbox.length !== 4 ||
@ -90,7 +101,27 @@ export async function POST(req: NextRequest) {
validBbox = [minLng, minLat, maxLng, maxLat]; validBbox = [minLng, minLat, maxLng, maxLat];
} }
if (validBbox) { if (boundary !== undefined) {
const boundaryJson = JSON.stringify(boundary);
await Promise.resolve(sql`
INSERT INTO cities (slug, name, country_code, geofabrik_url, bbox, boundary, status)
VALUES (
${slug as string},
${(name as string) ?? slug},
${(countryCode as string) ?? ""},
${geofabrikUrl},
ST_Envelope(ST_GeomFromGeoJSON(${boundaryJson})),
ST_Multi(ST_SetSRID(ST_GeomFromGeoJSON(${boundaryJson}), 4326)),
'pending'
)
ON CONFLICT (slug) DO UPDATE
SET status = 'pending',
geofabrik_url = EXCLUDED.geofabrik_url,
bbox = EXCLUDED.bbox,
boundary = EXCLUDED.boundary,
error_message = NULL
`);
} else if (validBbox) {
const [minLng, minLat, maxLng, maxLat] = validBbox; const [minLng, minLat, maxLng, maxLat] = validBbox;
await Promise.resolve(sql` await Promise.resolve(sql`
INSERT INTO cities (slug, name, country_code, geofabrik_url, bbox, status) INSERT INTO cities (slug, name, country_code, geofabrik_url, bbox, status)

129
apps/web/app/api/admin/jobs/[id]/stream/route.ts
View file

@ -2,8 +2,9 @@ import { NextRequest } from "next/server";
import { Job } from "bullmq"; import { Job } from "bullmq";
import { getPipelineQueue, getValhallaQueue } from "@/lib/queue"; import { getPipelineQueue, getValhallaQueue } from "@/lib/queue";
import type { PipelineJobData, JobProgress, ComputeScoresJobData, RefreshCityJobData } from "@/lib/queue"; import type { PipelineJobData, JobProgress, ComputeScoresJobData, RefreshCityJobData } from "@/lib/queue";
import type { SSEEvent } from "@transportationer/shared"; import type { SSEEvent, RoutingDetail } from "@transportationer/shared";
import { CATEGORY_IDS } from "@transportationer/shared"; import { CATEGORY_IDS } from "@transportationer/shared";
import { sql } from "@/lib/db";
export const runtime = "nodejs"; export const runtime = "nodejs";
@ -79,7 +80,7 @@ export async function GET(
const [pipelineActive, valhallaActive, waitingChildren] = await Promise.all([ const [pipelineActive, valhallaActive, waitingChildren] = await Promise.all([
queue.getActive(0, 100), queue.getActive(0, 100),
valhallaQueue.getActive(0, 100), valhallaQueue.getActive(0, 100),
queue.getWaitingChildren(0, 100), queue.getWaitingChildren(0, 200),
]); ]);
// 1a. Parallel routing phase: compute-scores is waiting for its routing // 1a. Parallel routing phase: compute-scores is waiting for its routing
@ -87,8 +88,6 @@ export async function GET(
// Only enter this branch when routingDispatched=true (Phase 1 has run). // Only enter this branch when routingDispatched=true (Phase 1 has run).
// Before that, compute-scores is in waiting-children while generate-grid // Before that, compute-scores is in waiting-children while generate-grid
// is running — fall through to the sequential active-job check instead. // is running — fall through to the sequential active-job check instead.
// Match by job ID (exact) when available; fall back to citySlug for the
// brief window before computeScoresJobId is written to the job record.
const csWaiting = waitingChildren.find( const csWaiting = waitingChildren.find(
(j) => (j) =>
j.data.type === "compute-scores" && j.data.type === "compute-scores" &&
@ -97,55 +96,95 @@ export async function GET(
); );
if (csWaiting) { if (csWaiting) {
const csData = csWaiting.data as ComputeScoresJobData; const csData = csWaiting.data as ComputeScoresJobData;
// Transit uses a single compute-transit child, not per-category routing jobs.
const routingModes = csData.modes.filter((m) => m !== "transit"); const routingModes = csData.modes.filter((m) => m !== "transit");
const totalRoutingJobs = routingModes.length * CATEGORY_IDS.length; const totalRoutingJobs = routingModes.length * CATEGORY_IDS.length;
const hasTransit = csData.modes.includes("transit"); const hasTransit = csData.modes.includes("transit");
// Count jobs that haven't finished yet (active or still waiting in queue)
const pipelineWaiting = await queue.getWaiting(0, 200); const pipelineWaiting = await queue.getWaiting(0, 200);
const stillRoutingActive = pipelineActive.filter(
(j) => j.data.citySlug === citySlug && j.data.type === "compute-routing", // Build per-mode routing detail for the UI
).length; const routingDetail: RoutingDetail = {};
const stillRoutingWaiting = pipelineWaiting.filter( for (const mode of routingModes) {
(j) => j.data.citySlug === citySlug && j.data.type === "compute-routing", const total = CATEGORY_IDS.length;
).length; const stillActive = pipelineActive.filter(
const completedRouting = Math.max(0, totalRoutingJobs - stillRoutingActive - stillRoutingWaiting); (j) => j.data.citySlug === citySlug && j.data.type === "compute-routing" && (j.data as any).mode === mode,
).length;
const stillWaiting = pipelineWaiting.filter(
(j) => j.data.citySlug === citySlug && j.data.type === "compute-routing" && (j.data as any).mode === mode,
).length;
routingDetail[mode] = { done: Math.max(0, total - stillActive - stillWaiting), total };
}
const completedRouting = Object.values(routingDetail).reduce((s, v) => s + v.done, 0);
// Check if compute-transit is still running // Check if compute-transit is still running
const transitRunning = const transitActive = hasTransit && pipelineActive.some(
hasTransit && (j) => j.data.citySlug === citySlug && j.data.type === "compute-transit",
(pipelineActive.some((j) => j.data.citySlug === citySlug && j.data.type === "compute-transit") || );
pipelineWaiting.some((j) => j.data.citySlug === citySlug && j.data.type === "compute-transit")); const transitWaiting = hasTransit && pipelineWaiting.some(
(j) => j.data.citySlug === citySlug && j.data.type === "compute-transit",
);
const transitRunning = transitActive || transitWaiting;
// compute-transit job also shows its own progress when active — prefer that if (hasTransit) {
routingDetail["transit"] = { done: transitRunning ? 0 : 1, total: 1 };
}
// Prefer reporting transit job's own progress when it's active
const transitActiveJob = pipelineActive.find( const transitActiveJob = pipelineActive.find(
(j) => j.data.citySlug === citySlug && j.data.type === "compute-transit", (j) => j.data.citySlug === citySlug && j.data.type === "compute-transit",
); );
if (transitActiveJob) { if (transitActiveJob) {
const p = transitActiveJob.progress as JobProgress | undefined; const p = transitActiveJob.progress as JobProgress | undefined;
if (p?.stage) { if (p?.stage) {
enqueue({ type: "progress", stage: p.stage, pct: p.pct, message: p.message }); enqueue({ type: "progress", stage: p.stage, pct: p.pct, message: p.message, routingDetail });
return; return;
} }
} }
const pct = totalRoutingJobs > 0 const pct = totalRoutingJobs > 0
? Math.round((completedRouting / totalRoutingJobs) * 100) ? Math.round((completedRouting / totalRoutingJobs) * 100)
: transitRunning ? 99 : 100; : transitRunning ? 50 : 100;
const message = transitRunning && completedRouting >= totalRoutingJobs const message = transitRunning && completedRouting >= totalRoutingJobs
? "Routing done — computing transit isochrones…" ? "Routing done — computing transit isochrones…"
: `${completedRouting} / ${totalRoutingJobs} routing jobs`; : `${completedRouting} / ${totalRoutingJobs} routing jobs`;
enqueue({ type: "progress", stage: "Computing scores", pct, message }); enqueue({ type: "progress", stage: "Computing scores", pct, message, routingDetail });
return; return;
} }
// 1b. Sequential phase: report whichever single job is currently active. // 1b. OSM parallel phase: extract-pois and build-valhalla run concurrently.
const activeJob = [...pipelineActive, ...valhallaActive].find( // Emit a separate progress event for each so the UI can track them independently.
const extractPoisJob = pipelineActive.find(
(j) => j.data.citySlug === citySlug && j.data.type === "extract-pois",
);
const buildValhallaJob = valhallaActive.find(
(j) => j.data.citySlug === citySlug && j.data.type === "build-valhalla",
);
if (extractPoisJob || buildValhallaJob) {
const pe = extractPoisJob?.progress as JobProgress | undefined;
const pv = buildValhallaJob?.progress as JobProgress | undefined;
if (pe?.stage) enqueue({ type: "progress", stage: pe.stage, pct: pe.pct, message: pe.message });
if (pv?.stage) enqueue({ type: "progress", stage: pv.stage, pct: pv.pct, message: pv.message });
if (!pe?.stage && !pv?.stage) enqueue({ type: "heartbeat" });
return;
}
// 1c. Sequential phase: report whichever single job is currently active.
// Two download-pbf jobs are enqueued per city (one child of extract-pois,
// one child of build-valhalla). The idempotency guard makes one skip
// immediately at pct=100 while the other does the real download.
// Prefer the job that has actual byte progress so the UI doesn't
// regress from 100% → 5% when the skip job is seen first.
const allCityActive = [...pipelineActive, ...valhallaActive].filter(
(j) => j.data.citySlug === citySlug && j.data.type !== "refresh-city", (j) => j.data.citySlug === citySlug && j.data.type !== "refresh-city",
); );
const activeJob =
allCityActive.find(
(j) =>
j.data.type === "download-pbf" &&
((j.progress as JobProgress | undefined)?.bytesDownloaded ?? 0) > 0,
) ?? allCityActive[0];
if (activeJob) { if (activeJob) {
const p = activeJob.progress as JobProgress | undefined; const p = activeJob.progress as JobProgress | undefined;
if (p?.stage) { if (p?.stage) {
@ -163,7 +202,30 @@ export async function GET(
return; return;
} }
// 2. No active stage — check for a failure that occurred after this refresh started. // 2. Check city status in DB — authoritative ground truth for completion/error.
// This catches completion reliably regardless of BullMQ job retention,
// and catches errors set by the worker's failed-job handler.
const cityRows = await Promise.resolve(sql<{ status: string; error_message: string | null }[]>`
SELECT status, error_message FROM cities WHERE slug = ${citySlug}
`);
const cityStatus = cityRows[0]?.status;
if (cityStatus === "ready") {
enqueue({ type: "completed", jobId: computeScoresJobId ?? id });
cleanup();
return;
}
if (cityStatus === "error") {
enqueue({
type: "failed",
jobId: id,
error: cityRows[0]?.error_message ?? "Pipeline failed",
});
cleanup();
return;
}
// 3. Check BullMQ failed queue as a secondary signal (catches failures
// before the worker's DB update propagates, e.g. DB connection issues).
const [pipelineFailed, valhallaFailed] = await Promise.all([ const [pipelineFailed, valhallaFailed] = await Promise.all([
queue.getFailed(0, 50), queue.getFailed(0, 50),
valhallaQueue.getFailed(0, 50), valhallaQueue.getFailed(0, 50),
@ -184,23 +246,6 @@ export async function GET(
return; return;
} }
// 3. Check if the specific compute-scores job completed → pipeline done.
// Use exact job ID match (computeScoresJobId) to avoid false positives
// from a previous run's completed record still in BullMQ's retention window.
const completed = await queue.getCompleted(0, 100);
const finalDone = completed.find((j) =>
computeScoresJobId
? j.id === computeScoresJobId
: j.data.citySlug === citySlug &&
j.data.type === "compute-scores" &&
(j.finishedOn ?? 0) > jobCreatedAt,
);
if (finalDone) {
enqueue({ type: "completed", jobId: finalDone.id ?? "" });
cleanup();
return;
}
// 4. Still pending — heartbeat. // 4. Still pending — heartbeat.
enqueue({ type: "heartbeat" }); enqueue({ type: "heartbeat" });
} catch { } catch {

5
apps/web/app/api/cities/route.ts
View file

@ -6,7 +6,7 @@ import type { City } from "@transportationer/shared";
export const runtime = "nodejs"; export const runtime = "nodejs";
export async function GET() { export async function GET() {
const cacheKey = "api:cities:all"; const cacheKey = "api:cities:all:v2";
const cached = await cacheGet<City[]>(cacheKey); const cached = await cacheGet<City[]>(cacheKey);
if (cached) { if (cached) {
return NextResponse.json(cached, { headers: { "X-Cache": "HIT" } }); return NextResponse.json(cached, { headers: { "X-Cache": "HIT" } });
@ -18,6 +18,7 @@ export async function GET() {
country_code: string; country_code: string;
geofabrik_url: string; geofabrik_url: string;
bbox_arr: number[] | null; bbox_arr: number[] | null;
boundary_geojson: string | null;
status: string; status: string;
last_ingested: string | null; last_ingested: string | null;
}[]>` }[]>`
@ -32,6 +33,7 @@ export async function GET() {
ST_XMax(bbox)::float, ST_XMax(bbox)::float,
ST_YMax(bbox)::float ST_YMax(bbox)::float
] END AS bbox_arr, ] END AS bbox_arr,
ST_AsGeoJSON(boundary) AS boundary_geojson,
status, status,
last_ingested last_ingested
FROM cities FROM cities
@ -44,6 +46,7 @@ export async function GET() {
countryCode: r.country_code, countryCode: r.country_code,
geofabrikUrl: r.geofabrik_url, geofabrikUrl: r.geofabrik_url,
bbox: (r.bbox_arr as [number, number, number, number]) ?? [0, 0, 0, 0], bbox: (r.bbox_arr as [number, number, number, number]) ?? [0, 0, 0, 0],
boundary: r.boundary_geojson ? JSON.parse(r.boundary_geojson) : null,
status: r.status as City["status"], status: r.status as City["status"],
lastIngested: r.last_ingested, lastIngested: r.last_ingested,
})); }));

8
apps/web/app/api/stats/route.ts
View file

@ -29,7 +29,11 @@ export async function GET(req: NextRequest) {
GROUP BY category GROUP BY category
`), `),
Promise.resolve(sql<{ count: number }[]>` Promise.resolve(sql<{ count: number }[]>`
SELECT COUNT(*)::int AS count FROM grid_points WHERE city_slug = ${city} SELECT COUNT(*)::int AS count
FROM grid_points gp
JOIN cities c ON c.slug = ${city}
WHERE gp.city_slug = ${city}
AND (c.boundary IS NULL OR ST_Within(gp.geom, c.boundary))
`), `),
Promise.resolve(sql<{ Promise.resolve(sql<{
category: string; category: string;
@ -46,7 +50,9 @@ export async function GET(req: NextRequest) {
(COUNT(*) FILTER (WHERE gs.score >= 0.5) * 100.0 / COUNT(*))::float AS coverage_pct (COUNT(*) FILTER (WHERE gs.score >= 0.5) * 100.0 / COUNT(*))::float AS coverage_pct
FROM grid_scores gs FROM grid_scores gs
JOIN grid_points gp ON gp.id = gs.grid_point_id JOIN grid_points gp ON gp.id = gs.grid_point_id
JOIN cities c ON c.slug = ${city}
WHERE gp.city_slug = ${city} WHERE gp.city_slug = ${city}
AND (c.boundary IS NULL OR ST_Within(gp.geom, c.boundary))
AND gs.travel_mode = ${mode} AND gs.travel_mode = ${mode}
AND gs.threshold_min = ${threshold} AND gs.threshold_min = ${threshold}
AND gs.profile = ${profile} AND gs.profile = ${profile}

2
apps/web/app/page.tsx
View file

@ -61,6 +61,7 @@ export default function HomePage() {
// Derived city data — used in effects below so must be declared before them // Derived city data — used in effects below so must be declared before them
const selectedCityData = cities.find((c) => c.slug === selectedCity); const selectedCityData = cities.find((c) => c.slug === selectedCity);
const cityBbox = selectedCityData?.bbox; const cityBbox = selectedCityData?.bbox;
const cityBoundary = selectedCityData?.boundary ?? null;
const estateValueAvailable = const estateValueAvailable =
cityBbox != null && cityBbox != null &&
cityBbox[0] < 11.779 && cityBbox[0] < 11.779 &&
@ -335,6 +336,7 @@ export default function HomePage() {
isochrones={overlayMode === "isochrone" ? isochroneData : null} isochrones={overlayMode === "isochrone" ? isochroneData : null}
baseOverlay={baseOverlay} baseOverlay={baseOverlay}
reachablePois={showPois ? reachablePois : []} reachablePois={showPois ? reachablePois : []}
cityBoundary={cityBoundary}
onLocationClick={handleLocationClick} onLocationClick={handleLocationClick}
/> />
)} )}

37
apps/web/components/map-view.tsx
View file

@ -43,6 +43,8 @@ export interface MapViewProps {
baseOverlay?: BaseOverlay; baseOverlay?: BaseOverlay;
/** Reachable POI pins to show on the map. */ /** Reachable POI pins to show on the map. */
reachablePois?: ReachablePoi[] | null; reachablePois?: ReachablePoi[] | null;
/** City administrative boundary polygon to outline on the map. */
cityBoundary?: object | null;
onLocationClick?: (lat: number, lng: number, estateValue: number | null) => void; onLocationClick?: (lat: number, lng: number, estateValue: number | null) => void;
} }
@ -166,6 +168,7 @@ export function MapView({
isochrones, isochrones,
baseOverlay = "accessibility", baseOverlay = "accessibility",
reachablePois, reachablePois,
cityBoundary,
onLocationClick, onLocationClick,
}: MapViewProps) { }: MapViewProps) {
const containerRef = useRef<HTMLDivElement>(null); const containerRef = useRef<HTMLDivElement>(null);
@ -575,6 +578,22 @@ export function MapView({
}; };
}, [mapLoaded, reachablePois]); }, [mapLoaded, reachablePois]);
// ── City boundary outline ─────────────────────────────────────────────────
useEffect(() => {
if (!mapLoaded) return;
const src = mapRef.current?.getSource("city-boundary") as maplibregl.GeoJSONSource | undefined;
if (!src) return;
src.setData(
cityBoundary
? {
type: "FeatureCollection",
// eslint-disable-next-line @typescript-eslint/no-explicit-any
features: [{ type: "Feature", geometry: cityBoundary as any, properties: {} }],
}
: { type: "FeatureCollection", features: [] },
);
}, [mapLoaded, cityBoundary]);
// ── Initialize map (runs once on mount) ─────────────────────────────────── // ── Initialize map (runs once on mount) ───────────────────────────────────
useEffect(() => { useEffect(() => {
if (mountedRef.current || !containerRef.current) return; if (mountedRef.current || !containerRef.current) return;
@ -615,6 +634,24 @@ export function MapView({
}, },
}); });
// City boundary outline — persistent layer, updated reactively via setData
map.addSource("city-boundary", {
type: "geojson",
data: { type: "FeatureCollection", features: [] },
});
map.addLayer({
id: "city-boundary-line",
type: "line",
source: "city-boundary",
paint: {
"line-color": "#1e293b",
"line-width": 1.5,
"line-opacity": 0.55,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
"line-dasharray": [5, 3] as any,
},
});
map.on("click", (e) => { map.on("click", (e) => {
const evFeatures = map.getLayer("estate-value-fill") const evFeatures = map.getLayer("estate-value-fill")
? map.queryRenderedFeatures(e.point, { layers: ["estate-value-fill"] }) ? map.queryRenderedFeatures(e.point, { layers: ["estate-value-fill"] })

121
apps/web/hooks/use-job-progress.ts
View file

@ -1,15 +1,26 @@
"use client"; "use client";
import { useEffect, useReducer, useRef } from "react"; import { useEffect, useReducer, useRef } from "react";
import type { SSEEvent } from "@transportationer/shared"; import type { SSEEvent, RoutingDetail } from "@transportationer/shared";
export type { RoutingDetail };
export type PipelineStageKey = // ─── Stage specification ──────────────────────────────────────────────────────
| "download-pbf"
| "extract-pois" interface StageSpec {
| "generate-grid" key: string;
| "build-valhalla" label: string;
| "compute-scores" /** Stages sharing the same parallelGroup run concurrently. */
| "refresh-city"; parallelGroup?: string;
}
export const STAGE_SPECS: StageSpec[] = [
{ key: "Downloading PBF", label: "Download OSM & GTFS data" },
{ key: "Extract POIs", label: "Extract & import POIs", parallelGroup: "osm" },
{ key: "Building routing graph",label: "Build routing graph", parallelGroup: "osm" },
{ key: "Generating grid", label: "Generate analysis grid" },
{ key: "Computing scores", label: "Compute accessibility" },
{ key: "Aggregating scores", label: "Aggregate scores" },
];
export interface StageStatus { export interface StageStatus {
key: string; key: string;
@ -17,58 +28,59 @@ export interface StageStatus {
status: "pending" | "active" | "completed" | "failed"; status: "pending" | "active" | "completed" | "failed";
pct: number; pct: number;
message: string; message: string;
parallelGroup?: string;
} }
// Four logical UI stages that map to the actual (parallel) pipeline jobs.
// extract-pois and build-valhalla run concurrently — they share "Processing OSM"
// so the linear mark-prior-as-completed logic stays correct.
const STAGE_ORDER: Array<{ key: string; label: string }> = [
{ key: "Downloading PBF", label: "Download OSM data" },
{ key: "Processing OSM", label: "Process OSM & build routes" },
{ key: "Generating grid", label: "Generate analysis grid" },
{ key: "Computing scores", label: "Compute accessibility scores" },
];
/** /**
* Maps raw worker stage strings → UI stage keys. * Maps raw worker stage strings → UI stage keys.
* All three parallel worker stages (extract-pois sub-stages + build-valhalla) * Returns null for stages that should be silently ignored
* fold into "Processing OSM". Routing sub-jobs and BORIS NI ingest fold * (e.g. the brief "Orchestrating pipeline" from refresh-city).
* into "Computing scores" (they run during compute-scores Phase 1).
*/ */
function normalizeStage(raw: string): string { function normalizeStage(raw: string): string | null {
if (raw === "Downloading GTFS") return "Downloading PBF";
if ( if (
raw === "Clipping to bounding box" || raw === "Clipping to bounding box" ||
raw === "Filtering OSM tags" || raw === "Filtering OSM tags" ||
raw === "Importing to PostGIS" || raw === "Importing to PostGIS"
raw === "Building routing graph" ) return "Extract POIs";
) return "Processing OSM"; // "Building routing graph" → direct key match
if (raw.startsWith("Routing ") || raw === "Ingesting BORIS NI") { // "Generating grid" → direct key match
return "Computing scores"; if (
} raw.startsWith("Routing ") ||
raw === "Transit routing" ||
raw === "Ingesting BORIS NI"
) return "Computing scores";
// "Computing scores" (Phase 1 brief dispatch update) → direct key match
// "Aggregating scores" (Phase 2) → direct key match
if (raw === "Orchestrating pipeline") return null; // ignore, handled by refresh-city
return raw; return raw;
} }
// ─── State ────────────────────────────────────────────────────────────────────
export type OverallStatus = "pending" | "active" | "completed" | "failed"; export type OverallStatus = "pending" | "active" | "completed" | "failed";
interface ProgressState { interface ProgressState {
stages: StageStatus[]; stages: StageStatus[];
overall: OverallStatus; overall: OverallStatus;
error?: string; error?: string;
routingDetail?: RoutingDetail;
} }
type Action = type Action =
| { type: "progress"; stage: string; pct: number; message: string } | { type: "progress"; stage: string; pct: number; message: string; routingDetail?: RoutingDetail }
| { type: "completed" } | { type: "completed" }
| { type: "failed"; error: string }; | { type: "failed"; error: string };
function initialState(): ProgressState { function initialState(): ProgressState {
return { return {
stages: STAGE_ORDER.map((s) => ({ stages: STAGE_SPECS.map((s) => ({
key: s.key, key: s.key,
label: s.label, label: s.label,
status: "pending", status: "pending",
pct: 0, pct: 0,
message: "", message: "",
parallelGroup: s.parallelGroup,
})), })),
overall: "pending", overall: "pending",
}; };
@ -78,45 +90,58 @@ function reducer(state: ProgressState, action: Action): ProgressState {
switch (action.type) { switch (action.type) {
case "progress": { case "progress": {
const stageKey = normalizeStage(action.stage); const stageKey = normalizeStage(action.stage);
if (stageKey === null) return { ...state, overall: "active" };
const stageIdx = STAGE_SPECS.findIndex((s) => s.key === stageKey);
if (stageIdx === -1) {
// Unknown stage — keep pipeline active but don't corrupt stage state
return { ...state, overall: "active" };
}
const activeGroup = STAGE_SPECS[stageIdx].parallelGroup;
let found = false; let found = false;
const stages = state.stages.map((s) => {
const stages = state.stages.map((s, i) => {
const spec = STAGE_SPECS[i];
if (s.key === stageKey) { if (s.key === stageKey) {
found = true; found = true;
return { return { ...s, status: "active" as const, pct: action.pct, message: action.message };
...s,
status: "active" as const,
pct: action.pct,
message: action.message,
};
} }
// Mark prior stages completed once a later stage is active // Don't auto-complete parallel siblings — they track their own lifecycle
if (activeGroup && spec.parallelGroup === activeGroup) return s;
// Mark all prior (non-sibling) stages as completed
if (!found) return { ...s, status: "completed" as const, pct: 100 }; if (!found) return { ...s, status: "completed" as const, pct: 100 };
return s; return s;
}); });
return { ...state, stages, overall: "active" };
return {
...state,
stages,
overall: "active",
routingDetail: action.routingDetail ?? state.routingDetail,
};
} }
case "completed": case "completed":
return { return {
...state, ...state,
overall: "completed", overall: "completed",
stages: state.stages.map((s) => ({ stages: state.stages.map((s) => ({ ...s, status: "completed", pct: 100 })),
...s,
status: "completed",
pct: 100,
})),
}; };
case "failed": case "failed":
return { ...state, overall: "failed", error: action.error }; return { ...state, overall: "failed", error: action.error };
default: default:
return state; return state;
} }
} }
// ─── Hook ─────────────────────────────────────────────────────────────────────
export function useJobProgress(jobId: string | null): ProgressState { export function useJobProgress(jobId: string | null): ProgressState {
const [state, dispatch] = useReducer(reducer, undefined, initialState); const [state, dispatch] = useReducer(reducer, undefined, initialState);
const esRef = useRef<EventSource | null>(null); const esRef = useRef<EventSource | null>(null);
// Tracks whether the stream ended with a legitimate "completed" event so
// the subsequent connection-close (which fires onerror) is ignored.
const completedRef = useRef(false); const completedRef = useRef(false);
useEffect(() => { useEffect(() => {
@ -130,8 +155,6 @@ export function useJobProgress(jobId: string | null): ProgressState {
const payload = JSON.parse(event.data) as SSEEvent; const payload = JSON.parse(event.data) as SSEEvent;
if (payload.type === "heartbeat") return; if (payload.type === "heartbeat") return;
if (payload.type === "completed") { if (payload.type === "completed") {
// Close before the server closes so the subsequent connection-close
// does not trigger onerror and overwrite the completed state.
completedRef.current = true; completedRef.current = true;
es.close(); es.close();
esRef.current = null; esRef.current = null;
@ -140,7 +163,7 @@ export function useJobProgress(jobId: string | null): ProgressState {
}; };
es.onerror = () => { es.onerror = () => {
if (completedRef.current) return; // Normal close after completion — ignore if (completedRef.current) return;
dispatch({ type: "failed", error: "Lost connection to job stream" }); dispatch({ type: "failed", error: "Lost connection to job stream" });
es.close(); es.close();
}; };

2
infra/schema.sql
View file

@ -21,8 +21,10 @@ CREATE TABLE IF NOT EXISTS cities (
-- Migration for existing databases -- Migration for existing databases
ALTER TABLE cities ADD COLUMN IF NOT EXISTS resolution_m INTEGER NOT NULL DEFAULT 200; ALTER TABLE cities ADD COLUMN IF NOT EXISTS resolution_m INTEGER NOT NULL DEFAULT 200;
ALTER TABLE cities ADD COLUMN IF NOT EXISTS boundary geometry(MultiPolygon, 4326);
CREATE INDEX IF NOT EXISTS idx_cities_bbox ON cities USING GIST (bbox); CREATE INDEX IF NOT EXISTS idx_cities_bbox ON cities USING GIST (bbox);
CREATE INDEX IF NOT EXISTS idx_cities_boundary ON cities USING GIST (boundary);
-- ─── Raw POIs (created and managed by osm2pgsql flex output) ───────────────── -- ─── Raw POIs (created and managed by osm2pgsql flex output) ─────────────────
-- osm2pgsql --drop recreates this table on each ingest using the Lua script. -- osm2pgsql --drop recreates this table on each ingest using the Lua script.

5
shared/src/types.ts
View file

@ -10,6 +10,7 @@ export interface City {
countryCode: string; countryCode: string;
geofabrikUrl: string; geofabrikUrl: string;
bbox: [number, number, number, number]; // [minLng, minLat, maxLng, maxLat] bbox: [number, number, number, number]; // [minLng, minLat, maxLng, maxLat]
boundary?: object | null;
status: CityStatus; status: CityStatus;
lastIngested: string | null; lastIngested: string | null;
} }
@ -108,8 +109,10 @@ export interface JobSummary {
// ─── SSE Events ────────────────────────────────────────────────────────────── // ─── SSE Events ──────────────────────────────────────────────────────────────
export type RoutingDetail = Record<string, { done: number; total: number }>;
export type SSEEvent = export type SSEEvent =
| { type: "progress"; stage: string; pct: number; message: string; bytesDownloaded?: number; totalBytes?: number } | { type: "progress"; stage: string; pct: number; message: string; bytesDownloaded?: number; totalBytes?: number; routingDetail?: RoutingDetail }
| { type: "completed"; jobId: string } | { type: "completed"; jobId: string }
| { type: "failed"; jobId: string; error: string } | { type: "failed"; jobId: string; error: string }
| { type: "heartbeat" }; | { type: "heartbeat" };

19
worker/src/index.ts
View file

@ -1,5 +1,6 @@
import { Worker, type Job } from "bullmq"; import { Worker, type Job } from "bullmq";
import { createBullMQConnection } from "./redis.js"; import { createBullMQConnection } from "./redis.js";
import { getSql } from "./db.js";
import type { PipelineJobData } from "@transportationer/shared"; import type { PipelineJobData } from "@transportationer/shared";
import { handleDownloadPbf } from "./jobs/download-pbf.js"; import { handleDownloadPbf } from "./jobs/download-pbf.js";
import { handleExtractPois } from "./jobs/extract-pois.js"; import { handleExtractPois } from "./jobs/extract-pois.js";
@ -54,8 +55,24 @@ worker.on("completed", (job) => {
console.log(`[worker] ✓ Job ${job.id} (${job.data.type}) completed`); console.log(`[worker] ✓ Job ${job.id} (${job.data.type}) completed`);
}); });
worker.on("failed", (job, err) => { worker.on("failed", async (job, err) => {
console.error(`[worker] ✗ Job ${job?.id} (${job?.data?.type}) failed:`, err.message); console.error(`[worker] ✗ Job ${job?.id} (${job?.data?.type}) failed:`, err.message);
// On final failure (all retries exhausted), mark the city as errored so
// the SSE stream and admin UI can reflect the true state immediately.
if (job?.data && "citySlug" in job.data && job.data.citySlug) {
const attemptsExhausted = job.attemptsMade >= (job.opts.attempts ?? 1);
if (attemptsExhausted) {
try {
const sql = getSql();
await sql`
UPDATE cities SET status = 'error', error_message = ${err.message.slice(0, 500)}
WHERE slug = ${job.data.citySlug}
`;
} catch (e) {
console.error("[worker] Failed to update city error status:", e);
}
}
}
}); });
worker.on("active", (job) => { worker.on("active", (job) => {

8
worker/src/jobs/compute-scores.ts
View file

@ -158,7 +158,7 @@ export async function handleComputeScores(
// ── Phase 2: aggregate scores from grid_poi_details ────────────────────── // ── Phase 2: aggregate scores from grid_poi_details ──────────────────────
await job.updateProgress({ await job.updateProgress({
stage: "Computing scores", stage: "Aggregating scores",
pct: 70, pct: 70,
message: `All routing complete — computing profile scores…`, message: `All routing complete — computing profile scores…`,
} satisfies JobProgress); } satisfies JobProgress);
@ -360,7 +360,7 @@ export async function handleComputeScores(
completedThresholds++; completedThresholds++;
await job.updateProgress({ await job.updateProgress({
stage: "Computing scores", stage: "Aggregating scores",
pct: 70 + Math.round((completedThresholds / thresholds.length) * 28), pct: 70 + Math.round((completedThresholds / thresholds.length) * 28),
message: `${completedThresholds} / ${thresholds.length} thresholds done…`, message: `${completedThresholds} / ${thresholds.length} thresholds done…`,
} satisfies JobProgress); } satisfies JobProgress);
@ -388,7 +388,7 @@ export async function handleComputeScores(
`); `);
if (n > 0) { if (n > 0) {
await job.updateProgress({ await job.updateProgress({
stage: "Computing scores", stage: "Aggregating scores",
pct: 99, pct: 99,
message: "Computing hidden gem scores…", message: "Computing hidden gem scores…",
} satisfies JobProgress); } satisfies JobProgress);
@ -436,7 +436,7 @@ export async function handleComputeScores(
} }
await job.updateProgress({ await job.updateProgress({
stage: "Computing scores", stage: "Aggregating scores",
pct: 100, pct: 100,
message: `All scores computed for ${citySlug}`, message: `All scores computed for ${citySlug}`,
} satisfies JobProgress); } satisfies JobProgress);