feat: add selection by city boundary, show that, stats only for city, not bbox
This commit is contained in:
parent
b891ca79ac
commit
e94d660686
12 changed files with 482 additions and 302 deletions
|
|
@ -3,6 +3,7 @@
|
|||
import { useState, useEffect, useMemo, useRef, useCallback } from "react";
|
||||
import type { GeofabrikFeature, GeofabrikIndex } from "@transportationer/shared";
|
||||
import { useJobProgress } from "@/hooks/use-job-progress";
|
||||
import type { StageStatus, RoutingDetail as RoutingDetailType } from "@/hooks/use-job-progress";
|
||||
|
||||
type Step = "browse" | "confirm" | "ingest";
|
||||
|
||||
|
|
@ -186,7 +187,7 @@ function GeofabrikBrowser({
|
|||
);
|
||||
}
|
||||
|
||||
// ─── Nominatim geocoder + radius selector + mini map ─────────────────────────
|
||||
// ─── Nominatim polygon geocoder + mini map ───────────────────────────────────
|
||||
|
||||
interface NominatimResult {
|
||||
place_id: number;
|
||||
|
|
@ -195,71 +196,34 @@ interface NominatimResult {
|
|||
lon: string;
|
||||
type: string;
|
||||
class: string;
|
||||
}
|
||||
|
||||
const RADIUS_OPTIONS = [2, 5, 10, 15, 20, 30];
|
||||
|
||||
function computeBbox(
|
||||
lat: number,
|
||||
lng: number,
|
||||
radiusKm: number,
|
||||
): [number, number, number, number] {
|
||||
const latDelta = radiusKm / 111.0;
|
||||
const lngDelta = radiusKm / (111.0 * Math.cos((lat * Math.PI) / 180));
|
||||
return [lng - lngDelta, lat - latDelta, lng + lngDelta, lat + latDelta];
|
||||
}
|
||||
|
||||
function bboxToGeoJSON(bbox: [number, number, number, number]) {
|
||||
const [w, s, e, n] = bbox;
|
||||
return {
|
||||
type: "Feature" as const,
|
||||
geometry: {
|
||||
type: "Polygon" as const,
|
||||
coordinates: [
|
||||
[
|
||||
[w, s],
|
||||
[e, s],
|
||||
[e, n],
|
||||
[w, n],
|
||||
[w, s],
|
||||
],
|
||||
],
|
||||
},
|
||||
properties: {},
|
||||
};
|
||||
geojson?: { type: string; coordinates: unknown };
|
||||
}
|
||||
|
||||
function LocationSelector({
|
||||
regionGeometry,
|
||||
onBboxChange,
|
||||
onBoundaryChange,
|
||||
}: {
|
||||
regionGeometry: GeofabrikFeature["geometry"];
|
||||
onBboxChange: (bbox: [number, number, number, number] | null) => void;
|
||||
onBoundaryChange: (boundary: { type: string; coordinates: unknown } | null) => void;
|
||||
}) {
|
||||
const [query, setQuery] = useState("");
|
||||
const [results, setResults] = useState<NominatimResult[]>([]);
|
||||
const [selected, setSelected] = useState<NominatimResult | null>(null);
|
||||
const [radius, setRadius] = useState(10);
|
||||
const [showDropdown, setShowDropdown] = useState(false);
|
||||
const [searching, setSearching] = useState(false);
|
||||
const mapContainerRef = useRef<HTMLDivElement>(null);
|
||||
const mapRef = useRef<import("maplibre-gl").Map | null>(null);
|
||||
const mapReadyRef = useRef(false);
|
||||
|
||||
const onBboxChangeRef = useRef(onBboxChange);
|
||||
onBboxChangeRef.current = onBboxChange;
|
||||
const onBoundaryChangeRef = useRef(onBoundaryChange);
|
||||
onBoundaryChangeRef.current = onBoundaryChange;
|
||||
|
||||
const bbox = useMemo((): [number, number, number, number] | null => {
|
||||
if (!selected) return null;
|
||||
return computeBbox(parseFloat(selected.lat), parseFloat(selected.lon), radius);
|
||||
}, [selected, radius]);
|
||||
|
||||
// Notify parent when bbox changes
|
||||
// Notify parent when selection changes
|
||||
useEffect(() => {
|
||||
onBboxChangeRef.current(bbox);
|
||||
}, [bbox]);
|
||||
onBoundaryChangeRef.current(selected?.geojson ?? null);
|
||||
}, [selected]);
|
||||
|
||||
// Debounced Nominatim search
|
||||
// Debounced Nominatim search — request polygon_geojson + featuretype=settlement
|
||||
useEffect(() => {
|
||||
if (query.length < 2) {
|
||||
setResults([]);
|
||||
|
|
@ -269,11 +233,15 @@ function LocationSelector({
|
|||
setSearching(true);
|
||||
try {
|
||||
const res = await fetch(
|
||||
`https://nominatim.openstreetmap.org/search?q=${encodeURIComponent(query)}&format=json&limit=6&addressdetails=0`,
|
||||
`https://nominatim.openstreetmap.org/search?q=${encodeURIComponent(query)}&polygon_geojson=1&featuretype=settlement&format=json&limit=8`,
|
||||
{ headers: { "User-Agent": "Transportationer/1.0 (15-minute city analyzer)" } },
|
||||
);
|
||||
const data: NominatimResult[] = await res.json();
|
||||
setResults(data);
|
||||
// Keep only results that have a real polygon boundary
|
||||
const polygons = data.filter(
|
||||
(r) => r.geojson?.type === "Polygon" || r.geojson?.type === "MultiPolygon",
|
||||
);
|
||||
setResults(polygons);
|
||||
setShowDropdown(true);
|
||||
} catch {
|
||||
setResults([]);
|
||||
|
|
@ -284,15 +252,27 @@ function LocationSelector({
|
|||
return () => clearTimeout(timer);
|
||||
}, [query]);
|
||||
|
||||
// Apply bbox to mini map
|
||||
const applyBboxToMap = useCallback(
|
||||
(map: import("maplibre-gl").Map, b: [number, number, number, number] | null) => {
|
||||
// Apply boundary polygon to mini map
|
||||
const applyBoundaryToMap = useCallback(
|
||||
(map: import("maplibre-gl").Map, geojson: NominatimResult["geojson"] | null) => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const src = map.getSource("area-bbox") as any;
|
||||
const src = map.getSource("area-boundary") as any;
|
||||
if (!src) return;
|
||||
if (b) {
|
||||
src.setData(bboxToGeoJSON(b));
|
||||
map.fitBounds([b[0], b[1], b[2], b[3]], { padding: 40, duration: 500 });
|
||||
if (geojson) {
|
||||
src.setData({ type: "Feature", geometry: geojson, properties: {} });
|
||||
// Fit to boundary bbox
|
||||
try {
|
||||
const coords: number[][] =
|
||||
geojson.type === "Polygon"
|
||||
? (geojson.coordinates as number[][][])[0]
|
||||
: (geojson.coordinates as number[][][][])[0][0];
|
||||
const lngs = coords.map((c) => c[0]);
|
||||
const lats = coords.map((c) => c[1]);
|
||||
map.fitBounds(
|
||||
[Math.min(...lngs), Math.min(...lats), Math.max(...lngs), Math.max(...lats)],
|
||||
{ padding: 40, duration: 500 },
|
||||
);
|
||||
} catch { /* ignore */ }
|
||||
} else {
|
||||
src.setData({ type: "FeatureCollection", features: [] });
|
||||
}
|
||||
|
|
@ -300,11 +280,11 @@ function LocationSelector({
|
|||
[],
|
||||
);
|
||||
|
||||
// Update mini map when bbox changes
|
||||
// Update mini map when selection changes
|
||||
useEffect(() => {
|
||||
const map = mapRef.current;
|
||||
if (map && mapReadyRef.current) applyBboxToMap(map, bbox);
|
||||
}, [bbox, applyBboxToMap]);
|
||||
if (map && mapReadyRef.current) applyBoundaryToMap(map, selected?.geojson ?? null);
|
||||
}, [selected, applyBoundaryToMap]);
|
||||
|
||||
// Initialize mini map
|
||||
useEffect(() => {
|
||||
|
|
@ -357,30 +337,26 @@ function LocationSelector({
|
|||
paint: { "line-color": "#64748b", "line-width": 1.5, "line-dasharray": [3, 2] },
|
||||
});
|
||||
|
||||
// Selected sub-area bbox
|
||||
map.addSource("area-bbox", {
|
||||
// Selected city boundary polygon
|
||||
map.addSource("area-boundary", {
|
||||
type: "geojson",
|
||||
data: { type: "FeatureCollection", features: [] },
|
||||
});
|
||||
map.addLayer({
|
||||
id: "area-bbox-fill",
|
||||
id: "area-boundary-fill",
|
||||
type: "fill",
|
||||
source: "area-bbox",
|
||||
source: "area-boundary",
|
||||
paint: { "fill-color": "#2563eb", "fill-opacity": 0.15 },
|
||||
});
|
||||
map.addLayer({
|
||||
id: "area-bbox-line",
|
||||
id: "area-boundary-line",
|
||||
type: "line",
|
||||
source: "area-bbox",
|
||||
source: "area-boundary",
|
||||
paint: { "line-color": "#2563eb", "line-width": 2 },
|
||||
});
|
||||
|
||||
// Fit to region if available
|
||||
if (regionGeometry) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const src = map.getSource("area-region") as any;
|
||||
if (src) {
|
||||
// Fit map to the region outline
|
||||
try {
|
||||
const coords: number[][] =
|
||||
regionGeometry.type === "Polygon"
|
||||
|
|
@ -392,14 +368,8 @@ function LocationSelector({
|
|||
[Math.min(...lngs), Math.min(...lats), Math.max(...lngs), Math.max(...lats)],
|
||||
{ padding: 20 },
|
||||
);
|
||||
} catch {
|
||||
/* ignore */
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply bbox if already set (e.g. after component re-render)
|
||||
if (bbox) applyBboxToMap(map, bbox);
|
||||
});
|
||||
})();
|
||||
|
||||
|
|
@ -418,7 +388,7 @@ function LocationSelector({
|
|||
<div className="flex gap-2 items-center">
|
||||
<input
|
||||
type="search"
|
||||
placeholder="Search for a city or location…"
|
||||
placeholder="Search for a city or municipality…"
|
||||
value={query}
|
||||
onChange={(e) => {
|
||||
setQuery(e.target.value);
|
||||
|
|
@ -451,35 +421,13 @@ function LocationSelector({
|
|||
<span className="text-gray-400 ml-1 text-xs">
|
||||
{r.display_name.split(",").slice(1, 3).join(",")}
|
||||
</span>
|
||||
<span className="ml-2 text-xs text-blue-500">{r.geojson?.type}</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Radius selector */}
|
||||
{selected && (
|
||||
<div className="flex items-center gap-3">
|
||||
<label className="text-sm text-gray-600 shrink-0">Radius:</label>
|
||||
<div className="flex gap-1.5 flex-wrap">
|
||||
{RADIUS_OPTIONS.map((r) => (
|
||||
<button
|
||||
key={r}
|
||||
type="button"
|
||||
onClick={() => setRadius(r)}
|
||||
className={`px-2.5 py-1 rounded-full text-xs font-medium border transition-colors ${
|
||||
radius === r
|
||||
? "bg-brand-600 text-white border-brand-600"
|
||||
: "bg-white text-gray-600 border-gray-300 hover:border-brand-400"
|
||||
}`}
|
||||
>
|
||||
{r} km
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Mini map */}
|
||||
<div
|
||||
ref={mapContainerRef}
|
||||
|
|
@ -487,14 +435,14 @@ function LocationSelector({
|
|||
style={{ height: 220 }}
|
||||
/>
|
||||
|
||||
{bbox && (
|
||||
{selected?.geojson && (
|
||||
<p className="text-xs text-green-700">
|
||||
✓ Sub-region: {radius} km around {selected!.display_name.split(",")[0]} — bbox [{bbox.map((v) => v.toFixed(4)).join(", ")}]
|
||||
✓ Boundary: {selected.display_name.split(",").slice(0, 2).join(", ")} ({selected.geojson.type})
|
||||
</p>
|
||||
)}
|
||||
{!selected && (
|
||||
<p className="text-xs text-gray-400">
|
||||
Search for a location to select a sub-region, or leave empty to use the entire dataset.
|
||||
Search for a city to use its administrative boundary for stats coverage, or leave empty to use the entire dataset.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
|
@ -514,18 +462,18 @@ function ConfirmStep({
|
|||
slug: string,
|
||||
name: string,
|
||||
countryCode: string,
|
||||
bbox: [number, number, number, number] | null,
|
||||
boundary: { type: string; coordinates: unknown } | null,
|
||||
) => Promise<void>;
|
||||
}) {
|
||||
const defaultSlug = region.properties.id.replace(/\//g, "-");
|
||||
const [slug, setSlug] = useState(defaultSlug);
|
||||
const [name, setName] = useState(region.properties.name);
|
||||
const [countryCode, setCountryCode] = useState("");
|
||||
const [bbox, setBbox] = useState<[number, number, number, number] | null>(null);
|
||||
const [boundary, setBoundary] = useState<{ type: string; coordinates: unknown } | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
const handleBboxChange = useCallback(
|
||||
(b: [number, number, number, number] | null) => setBbox(b),
|
||||
const handleBoundaryChange = useCallback(
|
||||
(b: { type: string; coordinates: unknown } | null) => setBoundary(b),
|
||||
[],
|
||||
);
|
||||
|
||||
|
|
@ -572,20 +520,20 @@ function ConfirmStep({
|
|||
/>
|
||||
</div>
|
||||
|
||||
{/* Sub-region selector */}
|
||||
{/* City boundary selector */}
|
||||
<div>
|
||||
<label className="block text-sm font-medium text-gray-700 mb-2">
|
||||
Sub-region{" "}
|
||||
<span className="text-gray-400 font-normal">(optional — clip to a city area)</span>
|
||||
City Boundary{" "}
|
||||
<span className="text-gray-400 font-normal">(optional — for accurate coverage stats)</span>
|
||||
</label>
|
||||
<p className="text-xs text-gray-500 mb-3">
|
||||
Select a city center and radius to ingest only that sub-area of the dataset.
|
||||
Recommended for large regions (e.g. entire states). Affects OSM import,
|
||||
routing tiles, and grid analysis.
|
||||
Search for the city municipality to use its administrative boundary for
|
||||
coverage statistics. The bounding box is still used for OSM import, routing,
|
||||
and grid generation.
|
||||
</p>
|
||||
<LocationSelector
|
||||
regionGeometry={region.geometry}
|
||||
onBboxChange={handleBboxChange}
|
||||
onBoundaryChange={handleBoundaryChange}
|
||||
/>
|
||||
</div>
|
||||
|
||||
|
|
@ -604,7 +552,7 @@ function ConfirmStep({
|
|||
<button
|
||||
onClick={async () => {
|
||||
setLoading(true);
|
||||
await onConfirm(slug, name, countryCode, bbox);
|
||||
await onConfirm(slug, name, countryCode, boundary);
|
||||
setLoading(false);
|
||||
}}
|
||||
disabled={loading || !slug || !name}
|
||||
|
|
@ -619,39 +567,147 @@ function ConfirmStep({
|
|||
|
||||
// ─── Progress step ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
function StageIcon({ status }: { status: StageStatus["status"] }) {
|
||||
if (status === "completed")
|
||||
return (
|
||||
<span className="w-5 h-5 flex items-center justify-center rounded-full bg-green-100 text-green-600 text-xs shrink-0">
|
||||
✓
|
||||
</span>
|
||||
);
|
||||
if (status === "failed")
|
||||
return (
|
||||
<span className="w-5 h-5 flex items-center justify-center rounded-full bg-red-100 text-red-600 text-xs shrink-0">
|
||||
✗
|
||||
</span>
|
||||
);
|
||||
if (status === "active")
|
||||
return (
|
||||
<span className="w-5 h-5 flex items-center justify-center shrink-0">
|
||||
<svg className="animate-spin w-4 h-4 text-brand-600" viewBox="0 0 24 24" fill="none">
|
||||
<circle cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="2" opacity="0.25" />
|
||||
<path d="M12 2a10 10 0 0 1 10 10" stroke="currentColor" strokeWidth="2" strokeLinecap="round" />
|
||||
</svg>
|
||||
</span>
|
||||
);
|
||||
return <span className="w-5 h-5 rounded-full border-2 border-gray-300 shrink-0" />;
|
||||
}
|
||||
|
||||
function StageRow({
|
||||
stage,
|
||||
error,
|
||||
}: {
|
||||
stage: StageStatus;
|
||||
error?: string;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex items-start gap-3">
|
||||
<StageIcon status={stage.status} />
|
||||
<div className="flex-1 min-w-0">
|
||||
<p className={`text-sm font-medium ${stage.status === "pending" ? "text-gray-400" : "text-gray-900"}`}>
|
||||
{stage.label}
|
||||
</p>
|
||||
{stage.status === "active" && (
|
||||
<>
|
||||
<div className="w-full bg-gray-200 rounded-full h-1 mt-1.5">
|
||||
<div
|
||||
className="bg-brand-600 h-1 rounded-full transition-all duration-500"
|
||||
style={{ width: `${stage.pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
<p className="text-xs text-gray-500 mt-0.5 truncate">{stage.message}</p>
|
||||
</>
|
||||
)}
|
||||
{stage.status === "failed" && error && (
|
||||
<p className="text-xs text-red-600 mt-0.5">{error}</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function RoutingGrid({ routingDetail }: { routingDetail: RoutingDetailType }) {
|
||||
const MODE_LABELS: Record<string, string> = {
|
||||
walking: "Walking",
|
||||
cycling: "Cycling",
|
||||
driving: "Driving",
|
||||
transit: "Transit",
|
||||
};
|
||||
const entries = Object.entries(routingDetail);
|
||||
if (entries.length === 0) return null;
|
||||
return (
|
||||
<div className="mt-2 space-y-1.5 pl-8">
|
||||
{entries.map(([mode, { done, total }]) => (
|
||||
<div key={mode} className="flex items-center gap-2">
|
||||
<span className="text-xs text-gray-500 w-14 shrink-0">{MODE_LABELS[mode] ?? mode}</span>
|
||||
<div className="flex-1 bg-gray-200 rounded-full h-1.5">
|
||||
<div
|
||||
className="bg-brand-500 h-1.5 rounded-full transition-all duration-500"
|
||||
style={{ width: total > 0 ? `${(done / total) * 100}%` : done > 0 ? "100%" : "0%" }}
|
||||
/>
|
||||
</div>
|
||||
<span className="text-xs text-gray-400 w-10 text-right shrink-0">
|
||||
{total > 1 ? `${done}/${total}` : done >= 1 ? "done" : "…"}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ProgressStep({ jobId }: { jobId: string | null }) {
|
||||
const { stages, overall, error } = useJobProgress(jobId);
|
||||
const { stages, overall, error, routingDetail } = useJobProgress(jobId);
|
||||
|
||||
// Group consecutive parallel stages together for rendering
|
||||
type StageGroup =
|
||||
| { kind: "single"; stage: StageStatus }
|
||||
| { kind: "parallel"; stages: StageStatus[] };
|
||||
|
||||
const groups: StageGroup[] = [];
|
||||
for (const stage of stages) {
|
||||
if (stage.parallelGroup) {
|
||||
const last = groups[groups.length - 1];
|
||||
if (last?.kind === "parallel" && last.stages[0].parallelGroup === stage.parallelGroup) {
|
||||
last.stages.push(stage);
|
||||
} else {
|
||||
groups.push({ kind: "parallel", stages: [stage] });
|
||||
}
|
||||
} else {
|
||||
groups.push({ kind: "single", stage });
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="card max-w-lg">
|
||||
<h2 className="text-lg font-semibold mb-6">Processing City Data</h2>
|
||||
|
||||
<ol className="space-y-4">
|
||||
{stages.map((stage) => (
|
||||
<li key={stage.key} className="flex items-start gap-3">
|
||||
<StageIcon status={stage.status} />
|
||||
<div className="flex-1 min-w-0">
|
||||
<p className="text-sm font-medium text-gray-900">{stage.label}</p>
|
||||
{stage.status === "active" && (
|
||||
<>
|
||||
<div className="w-full bg-gray-200 rounded-full h-1.5 mt-1.5">
|
||||
<div className="space-y-4">
|
||||
{groups.map((group, gi) =>
|
||||
group.kind === "single" ? (
|
||||
<div key={group.stage.key}>
|
||||
<StageRow stage={group.stage} error={error} />
|
||||
{/* Show per-mode routing grid under the compute-accessibility stage */}
|
||||
{group.stage.key === "Computing scores" &&
|
||||
group.stage.status === "active" &&
|
||||
routingDetail && (
|
||||
<RoutingGrid routingDetail={routingDetail} />
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<div
|
||||
className="bg-brand-600 h-1.5 rounded-full transition-all duration-500"
|
||||
style={{ width: `${stage.pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
<p className="text-xs text-gray-500 mt-1 truncate">
|
||||
{stage.message}
|
||||
key={`group-${gi}`}
|
||||
className="rounded-lg border border-gray-200 bg-gray-50 p-3 space-y-3"
|
||||
>
|
||||
<p className="text-xs font-medium text-gray-400 uppercase tracking-wide">
|
||||
Running in parallel
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
{stage.status === "failed" && error && (
|
||||
<p className="text-xs text-red-600 mt-1">{error}</p>
|
||||
{group.stages.map((s) => (
|
||||
<StageRow key={s.key} stage={s} error={error} />
|
||||
))}
|
||||
</div>
|
||||
),
|
||||
)}
|
||||
</div>
|
||||
</li>
|
||||
))}
|
||||
</ol>
|
||||
|
||||
{overall === "completed" && (
|
||||
<div className="mt-6 p-4 bg-green-50 rounded-lg text-green-800 text-sm">
|
||||
|
|
@ -669,7 +725,7 @@ function ProgressStep({ jobId }: { jobId: string | null }) {
|
|||
|
||||
{overall === "failed" && (
|
||||
<div className="mt-6 p-4 bg-red-50 rounded-lg text-red-800 text-sm">
|
||||
✗ Ingestion failed: {error}.{" "}
|
||||
✗ Ingestion failed: {error}{" "}
|
||||
<a href="/admin" className="underline">
|
||||
Return to dashboard
|
||||
</a>
|
||||
|
|
@ -680,51 +736,6 @@ function ProgressStep({ jobId }: { jobId: string | null }) {
|
|||
);
|
||||
}
|
||||
|
||||
function StageIcon({ status }: { status: StageStatus["status"] }) {
|
||||
if (status === "completed")
|
||||
return (
|
||||
<span className="w-5 h-5 flex items-center justify-center rounded-full bg-green-100 text-green-600 text-xs mt-0.5 shrink-0">
|
||||
✓
|
||||
</span>
|
||||
);
|
||||
if (status === "failed")
|
||||
return (
|
||||
<span className="w-5 h-5 flex items-center justify-center rounded-full bg-red-100 text-red-600 text-xs mt-0.5 shrink-0">
|
||||
✗
|
||||
</span>
|
||||
);
|
||||
if (status === "active")
|
||||
return (
|
||||
<span className="w-5 h-5 flex items-center justify-center mt-0.5 shrink-0">
|
||||
<svg
|
||||
className="animate-spin w-4 h-4 text-brand-600"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
>
|
||||
<circle
|
||||
cx="12"
|
||||
cy="12"
|
||||
r="10"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
opacity="0.25"
|
||||
/>
|
||||
<path
|
||||
d="M12 2a10 10 0 0 1 10 10"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
strokeLinecap="round"
|
||||
/>
|
||||
</svg>
|
||||
</span>
|
||||
);
|
||||
return (
|
||||
<span className="w-5 h-5 rounded-full border-2 border-gray-300 mt-0.5 shrink-0" />
|
||||
);
|
||||
}
|
||||
|
||||
import type { StageStatus } from "@/hooks/use-job-progress";
|
||||
|
||||
// ─── Main page ────────────────────────────────────────────────────────────────
|
||||
|
||||
export default function AddCityPage() {
|
||||
|
|
@ -737,7 +748,7 @@ export default function AddCityPage() {
|
|||
slug: string,
|
||||
name: string,
|
||||
countryCode: string,
|
||||
bbox: [number, number, number, number] | null,
|
||||
boundary: { type: string; coordinates: unknown } | null,
|
||||
) => {
|
||||
setIngestError(null);
|
||||
try {
|
||||
|
|
@ -749,7 +760,7 @@ export default function AddCityPage() {
|
|||
name,
|
||||
countryCode,
|
||||
geofabrikUrl: selected!.properties.urls.pbf,
|
||||
...(bbox ? { bbox } : {}),
|
||||
...(boundary ? { boundary } : {}),
|
||||
}),
|
||||
});
|
||||
if (!res.ok) {
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ export async function POST(req: NextRequest) {
|
|||
return NextResponse.json({ error: "Invalid JSON" }, { status: 400 });
|
||||
}
|
||||
|
||||
const { name, slug, countryCode, geofabrikUrl, resolutionM = 200, bbox } = body;
|
||||
const { name, slug, countryCode, geofabrikUrl, resolutionM = 200, bbox, boundary } = body;
|
||||
|
||||
if (typeof slug !== "string" || !/^[a-z0-9-]+$/.test(slug)) {
|
||||
return NextResponse.json(
|
||||
|
|
@ -67,9 +67,20 @@ export async function POST(req: NextRequest) {
|
|||
);
|
||||
}
|
||||
|
||||
// Validate optional boundary GeoJSON (Polygon or MultiPolygon)
|
||||
if (boundary !== undefined) {
|
||||
const b = boundary as Record<string, unknown>;
|
||||
if (b.type !== "Polygon" && b.type !== "MultiPolygon") {
|
||||
return NextResponse.json(
|
||||
{ error: "boundary must be a GeoJSON Polygon or MultiPolygon" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate optional bbox [minLng, minLat, maxLng, maxLat]
|
||||
let validBbox: [number, number, number, number] | null = null;
|
||||
if (bbox !== undefined) {
|
||||
if (bbox !== undefined && boundary === undefined) {
|
||||
if (
|
||||
!Array.isArray(bbox) ||
|
||||
bbox.length !== 4 ||
|
||||
|
|
@ -90,7 +101,27 @@ export async function POST(req: NextRequest) {
|
|||
validBbox = [minLng, minLat, maxLng, maxLat];
|
||||
}
|
||||
|
||||
if (validBbox) {
|
||||
if (boundary !== undefined) {
|
||||
const boundaryJson = JSON.stringify(boundary);
|
||||
await Promise.resolve(sql`
|
||||
INSERT INTO cities (slug, name, country_code, geofabrik_url, bbox, boundary, status)
|
||||
VALUES (
|
||||
${slug as string},
|
||||
${(name as string) ?? slug},
|
||||
${(countryCode as string) ?? ""},
|
||||
${geofabrikUrl},
|
||||
ST_Envelope(ST_GeomFromGeoJSON(${boundaryJson})),
|
||||
ST_Multi(ST_SetSRID(ST_GeomFromGeoJSON(${boundaryJson}), 4326)),
|
||||
'pending'
|
||||
)
|
||||
ON CONFLICT (slug) DO UPDATE
|
||||
SET status = 'pending',
|
||||
geofabrik_url = EXCLUDED.geofabrik_url,
|
||||
bbox = EXCLUDED.bbox,
|
||||
boundary = EXCLUDED.boundary,
|
||||
error_message = NULL
|
||||
`);
|
||||
} else if (validBbox) {
|
||||
const [minLng, minLat, maxLng, maxLat] = validBbox;
|
||||
await Promise.resolve(sql`
|
||||
INSERT INTO cities (slug, name, country_code, geofabrik_url, bbox, status)
|
||||
|
|
|
|||
|
|
@ -2,8 +2,9 @@ import { NextRequest } from "next/server";
|
|||
import { Job } from "bullmq";
|
||||
import { getPipelineQueue, getValhallaQueue } from "@/lib/queue";
|
||||
import type { PipelineJobData, JobProgress, ComputeScoresJobData, RefreshCityJobData } from "@/lib/queue";
|
||||
import type { SSEEvent } from "@transportationer/shared";
|
||||
import type { SSEEvent, RoutingDetail } from "@transportationer/shared";
|
||||
import { CATEGORY_IDS } from "@transportationer/shared";
|
||||
import { sql } from "@/lib/db";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
|
||||
|
|
@ -79,7 +80,7 @@ export async function GET(
|
|||
const [pipelineActive, valhallaActive, waitingChildren] = await Promise.all([
|
||||
queue.getActive(0, 100),
|
||||
valhallaQueue.getActive(0, 100),
|
||||
queue.getWaitingChildren(0, 100),
|
||||
queue.getWaitingChildren(0, 200),
|
||||
]);
|
||||
|
||||
// 1a. Parallel routing phase: compute-scores is waiting for its routing
|
||||
|
|
@ -87,8 +88,6 @@ export async function GET(
|
|||
// Only enter this branch when routingDispatched=true (Phase 1 has run).
|
||||
// Before that, compute-scores is in waiting-children while generate-grid
|
||||
// is running — fall through to the sequential active-job check instead.
|
||||
// Match by job ID (exact) when available; fall back to citySlug for the
|
||||
// brief window before computeScoresJobId is written to the job record.
|
||||
const csWaiting = waitingChildren.find(
|
||||
(j) =>
|
||||
j.data.type === "compute-scores" &&
|
||||
|
|
@ -97,55 +96,95 @@ export async function GET(
|
|||
);
|
||||
if (csWaiting) {
|
||||
const csData = csWaiting.data as ComputeScoresJobData;
|
||||
// Transit uses a single compute-transit child, not per-category routing jobs.
|
||||
const routingModes = csData.modes.filter((m) => m !== "transit");
|
||||
const totalRoutingJobs = routingModes.length * CATEGORY_IDS.length;
|
||||
const hasTransit = csData.modes.includes("transit");
|
||||
|
||||
// Count jobs that haven't finished yet (active or still waiting in queue)
|
||||
const pipelineWaiting = await queue.getWaiting(0, 200);
|
||||
const stillRoutingActive = pipelineActive.filter(
|
||||
(j) => j.data.citySlug === citySlug && j.data.type === "compute-routing",
|
||||
|
||||
// Build per-mode routing detail for the UI
|
||||
const routingDetail: RoutingDetail = {};
|
||||
for (const mode of routingModes) {
|
||||
const total = CATEGORY_IDS.length;
|
||||
const stillActive = pipelineActive.filter(
|
||||
(j) => j.data.citySlug === citySlug && j.data.type === "compute-routing" && (j.data as any).mode === mode,
|
||||
).length;
|
||||
const stillRoutingWaiting = pipelineWaiting.filter(
|
||||
(j) => j.data.citySlug === citySlug && j.data.type === "compute-routing",
|
||||
const stillWaiting = pipelineWaiting.filter(
|
||||
(j) => j.data.citySlug === citySlug && j.data.type === "compute-routing" && (j.data as any).mode === mode,
|
||||
).length;
|
||||
const completedRouting = Math.max(0, totalRoutingJobs - stillRoutingActive - stillRoutingWaiting);
|
||||
routingDetail[mode] = { done: Math.max(0, total - stillActive - stillWaiting), total };
|
||||
}
|
||||
|
||||
const completedRouting = Object.values(routingDetail).reduce((s, v) => s + v.done, 0);
|
||||
|
||||
// Check if compute-transit is still running
|
||||
const transitRunning =
|
||||
hasTransit &&
|
||||
(pipelineActive.some((j) => j.data.citySlug === citySlug && j.data.type === "compute-transit") ||
|
||||
pipelineWaiting.some((j) => j.data.citySlug === citySlug && j.data.type === "compute-transit"));
|
||||
const transitActive = hasTransit && pipelineActive.some(
|
||||
(j) => j.data.citySlug === citySlug && j.data.type === "compute-transit",
|
||||
);
|
||||
const transitWaiting = hasTransit && pipelineWaiting.some(
|
||||
(j) => j.data.citySlug === citySlug && j.data.type === "compute-transit",
|
||||
);
|
||||
const transitRunning = transitActive || transitWaiting;
|
||||
|
||||
// compute-transit job also shows its own progress when active — prefer that
|
||||
if (hasTransit) {
|
||||
routingDetail["transit"] = { done: transitRunning ? 0 : 1, total: 1 };
|
||||
}
|
||||
|
||||
// Prefer reporting transit job's own progress when it's active
|
||||
const transitActiveJob = pipelineActive.find(
|
||||
(j) => j.data.citySlug === citySlug && j.data.type === "compute-transit",
|
||||
);
|
||||
if (transitActiveJob) {
|
||||
const p = transitActiveJob.progress as JobProgress | undefined;
|
||||
if (p?.stage) {
|
||||
enqueue({ type: "progress", stage: p.stage, pct: p.pct, message: p.message });
|
||||
enqueue({ type: "progress", stage: p.stage, pct: p.pct, message: p.message, routingDetail });
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const pct = totalRoutingJobs > 0
|
||||
? Math.round((completedRouting / totalRoutingJobs) * 100)
|
||||
: transitRunning ? 99 : 100;
|
||||
: transitRunning ? 50 : 100;
|
||||
const message = transitRunning && completedRouting >= totalRoutingJobs
|
||||
? "Routing done — computing transit isochrones…"
|
||||
: `${completedRouting} / ${totalRoutingJobs} routing jobs`;
|
||||
|
||||
enqueue({ type: "progress", stage: "Computing scores", pct, message });
|
||||
enqueue({ type: "progress", stage: "Computing scores", pct, message, routingDetail });
|
||||
return;
|
||||
}
|
||||
|
||||
// 1b. Sequential phase: report whichever single job is currently active.
|
||||
const activeJob = [...pipelineActive, ...valhallaActive].find(
|
||||
// 1b. OSM parallel phase: extract-pois and build-valhalla run concurrently.
|
||||
// Emit a separate progress event for each so the UI can track them independently.
|
||||
const extractPoisJob = pipelineActive.find(
|
||||
(j) => j.data.citySlug === citySlug && j.data.type === "extract-pois",
|
||||
);
|
||||
const buildValhallaJob = valhallaActive.find(
|
||||
(j) => j.data.citySlug === citySlug && j.data.type === "build-valhalla",
|
||||
);
|
||||
if (extractPoisJob || buildValhallaJob) {
|
||||
const pe = extractPoisJob?.progress as JobProgress | undefined;
|
||||
const pv = buildValhallaJob?.progress as JobProgress | undefined;
|
||||
if (pe?.stage) enqueue({ type: "progress", stage: pe.stage, pct: pe.pct, message: pe.message });
|
||||
if (pv?.stage) enqueue({ type: "progress", stage: pv.stage, pct: pv.pct, message: pv.message });
|
||||
if (!pe?.stage && !pv?.stage) enqueue({ type: "heartbeat" });
|
||||
return;
|
||||
}
|
||||
|
||||
// 1c. Sequential phase: report whichever single job is currently active.
|
||||
// Two download-pbf jobs are enqueued per city (one child of extract-pois,
|
||||
// one child of build-valhalla). The idempotency guard makes one skip
|
||||
// immediately at pct=100 while the other does the real download.
|
||||
// Prefer the job that has actual byte progress so the UI doesn't
|
||||
// regress from 100% → 5% when the skip job is seen first.
|
||||
const allCityActive = [...pipelineActive, ...valhallaActive].filter(
|
||||
(j) => j.data.citySlug === citySlug && j.data.type !== "refresh-city",
|
||||
);
|
||||
|
||||
const activeJob =
|
||||
allCityActive.find(
|
||||
(j) =>
|
||||
j.data.type === "download-pbf" &&
|
||||
((j.progress as JobProgress | undefined)?.bytesDownloaded ?? 0) > 0,
|
||||
) ?? allCityActive[0];
|
||||
if (activeJob) {
|
||||
const p = activeJob.progress as JobProgress | undefined;
|
||||
if (p?.stage) {
|
||||
|
|
@ -163,7 +202,30 @@ export async function GET(
|
|||
return;
|
||||
}
|
||||
|
||||
// 2. No active stage — check for a failure that occurred after this refresh started.
|
||||
// 2. Check city status in DB — authoritative ground truth for completion/error.
|
||||
// This catches completion reliably regardless of BullMQ job retention,
|
||||
// and catches errors set by the worker's failed-job handler.
|
||||
const cityRows = await Promise.resolve(sql<{ status: string; error_message: string | null }[]>`
|
||||
SELECT status, error_message FROM cities WHERE slug = ${citySlug}
|
||||
`);
|
||||
const cityStatus = cityRows[0]?.status;
|
||||
if (cityStatus === "ready") {
|
||||
enqueue({ type: "completed", jobId: computeScoresJobId ?? id });
|
||||
cleanup();
|
||||
return;
|
||||
}
|
||||
if (cityStatus === "error") {
|
||||
enqueue({
|
||||
type: "failed",
|
||||
jobId: id,
|
||||
error: cityRows[0]?.error_message ?? "Pipeline failed",
|
||||
});
|
||||
cleanup();
|
||||
return;
|
||||
}
|
||||
|
||||
// 3. Check BullMQ failed queue as a secondary signal (catches failures
|
||||
// before the worker's DB update propagates, e.g. DB connection issues).
|
||||
const [pipelineFailed, valhallaFailed] = await Promise.all([
|
||||
queue.getFailed(0, 50),
|
||||
valhallaQueue.getFailed(0, 50),
|
||||
|
|
@ -184,23 +246,6 @@ export async function GET(
|
|||
return;
|
||||
}
|
||||
|
||||
// 3. Check if the specific compute-scores job completed → pipeline done.
|
||||
// Use exact job ID match (computeScoresJobId) to avoid false positives
|
||||
// from a previous run's completed record still in BullMQ's retention window.
|
||||
const completed = await queue.getCompleted(0, 100);
|
||||
const finalDone = completed.find((j) =>
|
||||
computeScoresJobId
|
||||
? j.id === computeScoresJobId
|
||||
: j.data.citySlug === citySlug &&
|
||||
j.data.type === "compute-scores" &&
|
||||
(j.finishedOn ?? 0) > jobCreatedAt,
|
||||
);
|
||||
if (finalDone) {
|
||||
enqueue({ type: "completed", jobId: finalDone.id ?? "" });
|
||||
cleanup();
|
||||
return;
|
||||
}
|
||||
|
||||
// 4. Still pending — heartbeat.
|
||||
enqueue({ type: "heartbeat" });
|
||||
} catch {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import type { City } from "@transportationer/shared";
|
|||
export const runtime = "nodejs";
|
||||
|
||||
export async function GET() {
|
||||
const cacheKey = "api:cities:all";
|
||||
const cacheKey = "api:cities:all:v2";
|
||||
const cached = await cacheGet<City[]>(cacheKey);
|
||||
if (cached) {
|
||||
return NextResponse.json(cached, { headers: { "X-Cache": "HIT" } });
|
||||
|
|
@ -18,6 +18,7 @@ export async function GET() {
|
|||
country_code: string;
|
||||
geofabrik_url: string;
|
||||
bbox_arr: number[] | null;
|
||||
boundary_geojson: string | null;
|
||||
status: string;
|
||||
last_ingested: string | null;
|
||||
}[]>`
|
||||
|
|
@ -32,6 +33,7 @@ export async function GET() {
|
|||
ST_XMax(bbox)::float,
|
||||
ST_YMax(bbox)::float
|
||||
] END AS bbox_arr,
|
||||
ST_AsGeoJSON(boundary) AS boundary_geojson,
|
||||
status,
|
||||
last_ingested
|
||||
FROM cities
|
||||
|
|
@ -44,6 +46,7 @@ export async function GET() {
|
|||
countryCode: r.country_code,
|
||||
geofabrikUrl: r.geofabrik_url,
|
||||
bbox: (r.bbox_arr as [number, number, number, number]) ?? [0, 0, 0, 0],
|
||||
boundary: r.boundary_geojson ? JSON.parse(r.boundary_geojson) : null,
|
||||
status: r.status as City["status"],
|
||||
lastIngested: r.last_ingested,
|
||||
}));
|
||||
|
|
|
|||
|
|
@ -29,7 +29,11 @@ export async function GET(req: NextRequest) {
|
|||
GROUP BY category
|
||||
`),
|
||||
Promise.resolve(sql<{ count: number }[]>`
|
||||
SELECT COUNT(*)::int AS count FROM grid_points WHERE city_slug = ${city}
|
||||
SELECT COUNT(*)::int AS count
|
||||
FROM grid_points gp
|
||||
JOIN cities c ON c.slug = ${city}
|
||||
WHERE gp.city_slug = ${city}
|
||||
AND (c.boundary IS NULL OR ST_Within(gp.geom, c.boundary))
|
||||
`),
|
||||
Promise.resolve(sql<{
|
||||
category: string;
|
||||
|
|
@ -46,7 +50,9 @@ export async function GET(req: NextRequest) {
|
|||
(COUNT(*) FILTER (WHERE gs.score >= 0.5) * 100.0 / COUNT(*))::float AS coverage_pct
|
||||
FROM grid_scores gs
|
||||
JOIN grid_points gp ON gp.id = gs.grid_point_id
|
||||
JOIN cities c ON c.slug = ${city}
|
||||
WHERE gp.city_slug = ${city}
|
||||
AND (c.boundary IS NULL OR ST_Within(gp.geom, c.boundary))
|
||||
AND gs.travel_mode = ${mode}
|
||||
AND gs.threshold_min = ${threshold}
|
||||
AND gs.profile = ${profile}
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ export default function HomePage() {
|
|||
// Derived city data — used in effects below so must be declared before them
|
||||
const selectedCityData = cities.find((c) => c.slug === selectedCity);
|
||||
const cityBbox = selectedCityData?.bbox;
|
||||
const cityBoundary = selectedCityData?.boundary ?? null;
|
||||
const estateValueAvailable =
|
||||
cityBbox != null &&
|
||||
cityBbox[0] < 11.779 &&
|
||||
|
|
@ -335,6 +336,7 @@ export default function HomePage() {
|
|||
isochrones={overlayMode === "isochrone" ? isochroneData : null}
|
||||
baseOverlay={baseOverlay}
|
||||
reachablePois={showPois ? reachablePois : []}
|
||||
cityBoundary={cityBoundary}
|
||||
onLocationClick={handleLocationClick}
|
||||
/>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,8 @@ export interface MapViewProps {
|
|||
baseOverlay?: BaseOverlay;
|
||||
/** Reachable POI pins to show on the map. */
|
||||
reachablePois?: ReachablePoi[] | null;
|
||||
/** City administrative boundary polygon to outline on the map. */
|
||||
cityBoundary?: object | null;
|
||||
onLocationClick?: (lat: number, lng: number, estateValue: number | null) => void;
|
||||
}
|
||||
|
||||
|
|
@ -166,6 +168,7 @@ export function MapView({
|
|||
isochrones,
|
||||
baseOverlay = "accessibility",
|
||||
reachablePois,
|
||||
cityBoundary,
|
||||
onLocationClick,
|
||||
}: MapViewProps) {
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
|
|
@ -575,6 +578,22 @@ export function MapView({
|
|||
};
|
||||
}, [mapLoaded, reachablePois]);
|
||||
|
||||
// ── City boundary outline ─────────────────────────────────────────────────
|
||||
useEffect(() => {
|
||||
if (!mapLoaded) return;
|
||||
const src = mapRef.current?.getSource("city-boundary") as maplibregl.GeoJSONSource | undefined;
|
||||
if (!src) return;
|
||||
src.setData(
|
||||
cityBoundary
|
||||
? {
|
||||
type: "FeatureCollection",
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
features: [{ type: "Feature", geometry: cityBoundary as any, properties: {} }],
|
||||
}
|
||||
: { type: "FeatureCollection", features: [] },
|
||||
);
|
||||
}, [mapLoaded, cityBoundary]);
|
||||
|
||||
// ── Initialize map (runs once on mount) ───────────────────────────────────
|
||||
useEffect(() => {
|
||||
if (mountedRef.current || !containerRef.current) return;
|
||||
|
|
@ -615,6 +634,24 @@ export function MapView({
|
|||
},
|
||||
});
|
||||
|
||||
// City boundary outline — persistent layer, updated reactively via setData
|
||||
map.addSource("city-boundary", {
|
||||
type: "geojson",
|
||||
data: { type: "FeatureCollection", features: [] },
|
||||
});
|
||||
map.addLayer({
|
||||
id: "city-boundary-line",
|
||||
type: "line",
|
||||
source: "city-boundary",
|
||||
paint: {
|
||||
"line-color": "#1e293b",
|
||||
"line-width": 1.5,
|
||||
"line-opacity": 0.55,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
"line-dasharray": [5, 3] as any,
|
||||
},
|
||||
});
|
||||
|
||||
map.on("click", (e) => {
|
||||
const evFeatures = map.getLayer("estate-value-fill")
|
||||
? map.queryRenderedFeatures(e.point, { layers: ["estate-value-fill"] })
|
||||
|
|
|
|||
|
|
@ -1,15 +1,26 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useReducer, useRef } from "react";
|
||||
import type { SSEEvent } from "@transportationer/shared";
|
||||
import type { SSEEvent, RoutingDetail } from "@transportationer/shared";
|
||||
export type { RoutingDetail };
|
||||
|
||||
export type PipelineStageKey =
|
||||
| "download-pbf"
|
||||
| "extract-pois"
|
||||
| "generate-grid"
|
||||
| "build-valhalla"
|
||||
| "compute-scores"
|
||||
| "refresh-city";
|
||||
// ─── Stage specification ──────────────────────────────────────────────────────
|
||||
|
||||
interface StageSpec {
|
||||
key: string;
|
||||
label: string;
|
||||
/** Stages sharing the same parallelGroup run concurrently. */
|
||||
parallelGroup?: string;
|
||||
}
|
||||
|
||||
export const STAGE_SPECS: StageSpec[] = [
|
||||
{ key: "Downloading PBF", label: "Download OSM & GTFS data" },
|
||||
{ key: "Extract POIs", label: "Extract & import POIs", parallelGroup: "osm" },
|
||||
{ key: "Building routing graph",label: "Build routing graph", parallelGroup: "osm" },
|
||||
{ key: "Generating grid", label: "Generate analysis grid" },
|
||||
{ key: "Computing scores", label: "Compute accessibility" },
|
||||
{ key: "Aggregating scores", label: "Aggregate scores" },
|
||||
];
|
||||
|
||||
export interface StageStatus {
|
||||
key: string;
|
||||
|
|
@ -17,58 +28,59 @@ export interface StageStatus {
|
|||
status: "pending" | "active" | "completed" | "failed";
|
||||
pct: number;
|
||||
message: string;
|
||||
parallelGroup?: string;
|
||||
}
|
||||
|
||||
// Four logical UI stages that map to the actual (parallel) pipeline jobs.
|
||||
// extract-pois and build-valhalla run concurrently — they share "Processing OSM"
|
||||
// so the linear mark-prior-as-completed logic stays correct.
|
||||
const STAGE_ORDER: Array<{ key: string; label: string }> = [
|
||||
{ key: "Downloading PBF", label: "Download OSM data" },
|
||||
{ key: "Processing OSM", label: "Process OSM & build routes" },
|
||||
{ key: "Generating grid", label: "Generate analysis grid" },
|
||||
{ key: "Computing scores", label: "Compute accessibility scores" },
|
||||
];
|
||||
|
||||
/**
|
||||
* Maps raw worker stage strings → UI stage keys.
|
||||
* All three parallel worker stages (extract-pois sub-stages + build-valhalla)
|
||||
* fold into "Processing OSM". Routing sub-jobs and BORIS NI ingest fold
|
||||
* into "Computing scores" (they run during compute-scores Phase 1).
|
||||
* Returns null for stages that should be silently ignored
|
||||
* (e.g. the brief "Orchestrating pipeline" from refresh-city).
|
||||
*/
|
||||
function normalizeStage(raw: string): string {
|
||||
function normalizeStage(raw: string): string | null {
|
||||
if (raw === "Downloading GTFS") return "Downloading PBF";
|
||||
if (
|
||||
raw === "Clipping to bounding box" ||
|
||||
raw === "Filtering OSM tags" ||
|
||||
raw === "Importing to PostGIS" ||
|
||||
raw === "Building routing graph"
|
||||
) return "Processing OSM";
|
||||
if (raw.startsWith("Routing ") || raw === "Ingesting BORIS NI") {
|
||||
return "Computing scores";
|
||||
}
|
||||
raw === "Importing to PostGIS"
|
||||
) return "Extract POIs";
|
||||
// "Building routing graph" → direct key match
|
||||
// "Generating grid" → direct key match
|
||||
if (
|
||||
raw.startsWith("Routing ") ||
|
||||
raw === "Transit routing" ||
|
||||
raw === "Ingesting BORIS NI"
|
||||
) return "Computing scores";
|
||||
// "Computing scores" (Phase 1 brief dispatch update) → direct key match
|
||||
// "Aggregating scores" (Phase 2) → direct key match
|
||||
if (raw === "Orchestrating pipeline") return null; // ignore, handled by refresh-city
|
||||
return raw;
|
||||
}
|
||||
|
||||
// ─── State ────────────────────────────────────────────────────────────────────
|
||||
|
||||
export type OverallStatus = "pending" | "active" | "completed" | "failed";
|
||||
|
||||
interface ProgressState {
|
||||
stages: StageStatus[];
|
||||
overall: OverallStatus;
|
||||
error?: string;
|
||||
routingDetail?: RoutingDetail;
|
||||
}
|
||||
|
||||
type Action =
|
||||
| { type: "progress"; stage: string; pct: number; message: string }
|
||||
| { type: "progress"; stage: string; pct: number; message: string; routingDetail?: RoutingDetail }
|
||||
| { type: "completed" }
|
||||
| { type: "failed"; error: string };
|
||||
|
||||
function initialState(): ProgressState {
|
||||
return {
|
||||
stages: STAGE_ORDER.map((s) => ({
|
||||
stages: STAGE_SPECS.map((s) => ({
|
||||
key: s.key,
|
||||
label: s.label,
|
||||
status: "pending",
|
||||
pct: 0,
|
||||
message: "",
|
||||
parallelGroup: s.parallelGroup,
|
||||
})),
|
||||
overall: "pending",
|
||||
};
|
||||
|
|
@ -78,45 +90,58 @@ function reducer(state: ProgressState, action: Action): ProgressState {
|
|||
switch (action.type) {
|
||||
case "progress": {
|
||||
const stageKey = normalizeStage(action.stage);
|
||||
if (stageKey === null) return { ...state, overall: "active" };
|
||||
|
||||
const stageIdx = STAGE_SPECS.findIndex((s) => s.key === stageKey);
|
||||
if (stageIdx === -1) {
|
||||
// Unknown stage — keep pipeline active but don't corrupt stage state
|
||||
return { ...state, overall: "active" };
|
||||
}
|
||||
|
||||
const activeGroup = STAGE_SPECS[stageIdx].parallelGroup;
|
||||
let found = false;
|
||||
const stages = state.stages.map((s) => {
|
||||
|
||||
const stages = state.stages.map((s, i) => {
|
||||
const spec = STAGE_SPECS[i];
|
||||
if (s.key === stageKey) {
|
||||
found = true;
|
||||
return {
|
||||
...s,
|
||||
status: "active" as const,
|
||||
pct: action.pct,
|
||||
message: action.message,
|
||||
};
|
||||
return { ...s, status: "active" as const, pct: action.pct, message: action.message };
|
||||
}
|
||||
// Mark prior stages completed once a later stage is active
|
||||
// Don't auto-complete parallel siblings — they track their own lifecycle
|
||||
if (activeGroup && spec.parallelGroup === activeGroup) return s;
|
||||
// Mark all prior (non-sibling) stages as completed
|
||||
if (!found) return { ...s, status: "completed" as const, pct: 100 };
|
||||
return s;
|
||||
});
|
||||
return { ...state, stages, overall: "active" };
|
||||
|
||||
return {
|
||||
...state,
|
||||
stages,
|
||||
overall: "active",
|
||||
routingDetail: action.routingDetail ?? state.routingDetail,
|
||||
};
|
||||
}
|
||||
|
||||
case "completed":
|
||||
return {
|
||||
...state,
|
||||
overall: "completed",
|
||||
stages: state.stages.map((s) => ({
|
||||
...s,
|
||||
status: "completed",
|
||||
pct: 100,
|
||||
})),
|
||||
stages: state.stages.map((s) => ({ ...s, status: "completed", pct: 100 })),
|
||||
};
|
||||
|
||||
case "failed":
|
||||
return { ...state, overall: "failed", error: action.error };
|
||||
|
||||
default:
|
||||
return state;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Hook ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
export function useJobProgress(jobId: string | null): ProgressState {
|
||||
const [state, dispatch] = useReducer(reducer, undefined, initialState);
|
||||
const esRef = useRef<EventSource | null>(null);
|
||||
// Tracks whether the stream ended with a legitimate "completed" event so
|
||||
// the subsequent connection-close (which fires onerror) is ignored.
|
||||
const completedRef = useRef(false);
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -130,8 +155,6 @@ export function useJobProgress(jobId: string | null): ProgressState {
|
|||
const payload = JSON.parse(event.data) as SSEEvent;
|
||||
if (payload.type === "heartbeat") return;
|
||||
if (payload.type === "completed") {
|
||||
// Close before the server closes so the subsequent connection-close
|
||||
// does not trigger onerror and overwrite the completed state.
|
||||
completedRef.current = true;
|
||||
es.close();
|
||||
esRef.current = null;
|
||||
|
|
@ -140,7 +163,7 @@ export function useJobProgress(jobId: string | null): ProgressState {
|
|||
};
|
||||
|
||||
es.onerror = () => {
|
||||
if (completedRef.current) return; // Normal close after completion — ignore
|
||||
if (completedRef.current) return;
|
||||
dispatch({ type: "failed", error: "Lost connection to job stream" });
|
||||
es.close();
|
||||
};
|
||||
|
|
|
|||
|
|
@ -21,8 +21,10 @@ CREATE TABLE IF NOT EXISTS cities (
|
|||
|
||||
-- Migration for existing databases
|
||||
ALTER TABLE cities ADD COLUMN IF NOT EXISTS resolution_m INTEGER NOT NULL DEFAULT 200;
|
||||
ALTER TABLE cities ADD COLUMN IF NOT EXISTS boundary geometry(MultiPolygon, 4326);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cities_bbox ON cities USING GIST (bbox);
|
||||
CREATE INDEX IF NOT EXISTS idx_cities_boundary ON cities USING GIST (boundary);
|
||||
|
||||
-- ─── Raw POIs (created and managed by osm2pgsql flex output) ─────────────────
|
||||
-- osm2pgsql --drop recreates this table on each ingest using the Lua script.
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ export interface City {
|
|||
countryCode: string;
|
||||
geofabrikUrl: string;
|
||||
bbox: [number, number, number, number]; // [minLng, minLat, maxLng, maxLat]
|
||||
boundary?: object | null;
|
||||
status: CityStatus;
|
||||
lastIngested: string | null;
|
||||
}
|
||||
|
|
@ -108,8 +109,10 @@ export interface JobSummary {
|
|||
|
||||
// ─── SSE Events ──────────────────────────────────────────────────────────────
|
||||
|
||||
export type RoutingDetail = Record<string, { done: number; total: number }>;
|
||||
|
||||
export type SSEEvent =
|
||||
| { type: "progress"; stage: string; pct: number; message: string; bytesDownloaded?: number; totalBytes?: number }
|
||||
| { type: "progress"; stage: string; pct: number; message: string; bytesDownloaded?: number; totalBytes?: number; routingDetail?: RoutingDetail }
|
||||
| { type: "completed"; jobId: string }
|
||||
| { type: "failed"; jobId: string; error: string }
|
||||
| { type: "heartbeat" };
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { Worker, type Job } from "bullmq";
|
||||
import { createBullMQConnection } from "./redis.js";
|
||||
import { getSql } from "./db.js";
|
||||
import type { PipelineJobData } from "@transportationer/shared";
|
||||
import { handleDownloadPbf } from "./jobs/download-pbf.js";
|
||||
import { handleExtractPois } from "./jobs/extract-pois.js";
|
||||
|
|
@ -54,8 +55,24 @@ worker.on("completed", (job) => {
|
|||
console.log(`[worker] ✓ Job ${job.id} (${job.data.type}) completed`);
|
||||
});
|
||||
|
||||
worker.on("failed", (job, err) => {
|
||||
worker.on("failed", async (job, err) => {
|
||||
console.error(`[worker] ✗ Job ${job?.id} (${job?.data?.type}) failed:`, err.message);
|
||||
// On final failure (all retries exhausted), mark the city as errored so
|
||||
// the SSE stream and admin UI can reflect the true state immediately.
|
||||
if (job?.data && "citySlug" in job.data && job.data.citySlug) {
|
||||
const attemptsExhausted = job.attemptsMade >= (job.opts.attempts ?? 1);
|
||||
if (attemptsExhausted) {
|
||||
try {
|
||||
const sql = getSql();
|
||||
await sql`
|
||||
UPDATE cities SET status = 'error', error_message = ${err.message.slice(0, 500)}
|
||||
WHERE slug = ${job.data.citySlug}
|
||||
`;
|
||||
} catch (e) {
|
||||
console.error("[worker] Failed to update city error status:", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
worker.on("active", (job) => {
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ export async function handleComputeScores(
|
|||
|
||||
// ── Phase 2: aggregate scores from grid_poi_details ──────────────────────
|
||||
await job.updateProgress({
|
||||
stage: "Computing scores",
|
||||
stage: "Aggregating scores",
|
||||
pct: 70,
|
||||
message: `All routing complete — computing profile scores…`,
|
||||
} satisfies JobProgress);
|
||||
|
|
@ -360,7 +360,7 @@ export async function handleComputeScores(
|
|||
|
||||
completedThresholds++;
|
||||
await job.updateProgress({
|
||||
stage: "Computing scores",
|
||||
stage: "Aggregating scores",
|
||||
pct: 70 + Math.round((completedThresholds / thresholds.length) * 28),
|
||||
message: `${completedThresholds} / ${thresholds.length} thresholds done…`,
|
||||
} satisfies JobProgress);
|
||||
|
|
@ -388,7 +388,7 @@ export async function handleComputeScores(
|
|||
`);
|
||||
if (n > 0) {
|
||||
await job.updateProgress({
|
||||
stage: "Computing scores",
|
||||
stage: "Aggregating scores",
|
||||
pct: 99,
|
||||
message: "Computing hidden gem scores…",
|
||||
} satisfies JobProgress);
|
||||
|
|
@ -436,7 +436,7 @@ export async function handleComputeScores(
|
|||
}
|
||||
|
||||
await job.updateProgress({
|
||||
stage: "Computing scores",
|
||||
stage: "Aggregating scores",
|
||||
pct: 100,
|
||||
message: `All scores computed for ${citySlug}`,
|
||||
} satisfies JobProgress);
|
||||
|
|
|
|||
Loading…
Reference in a new issue