feat: fetch boris ni data of previous years

apps/web/app/api/estate-value/route.ts

@@ -42,11 +42,24 @@ export async function GET(request: Request) {
         WITH pt AS (
           SELECT ST_SetSRID(ST_Point(${lng}, ${lat}), 4326) AS geom
         ),
+        -- Use only the latest year's data for this city so historical rows
+        -- don't create duplicate zone polygons on top of each other.
+        latest_year AS (
+          SELECT MAX(year) AS yr
+          FROM estate_value_zones
+          WHERE source = 'boris-ni'
+            AND city_slug = (
+              SELECT city_slug FROM estate_value_zones, pt
+              WHERE ST_Within(pt.geom, estate_value_zones.geom)
+              LIMIT 1
+            )
+        ),
         nearby AS (
-          SELECT value_eur_m2
+          SELECT ez.value_eur_m2
-          FROM estate_value_zones, pt
+          FROM estate_value_zones ez, pt, latest_year
-          WHERE value_eur_m2 IS NOT NULL
+          WHERE ez.value_eur_m2 IS NOT NULL
-            AND ST_DWithin(estate_value_zones.geom::geography, pt.geom::geography, 5000)
+            AND ST_DWithin(ez.geom::geography, pt.geom::geography, 5000)
+            AND (ez.year IS NULL OR ez.year = latest_year.yr)
         )
         SELECT
           ST_AsGeoJSON(ez.geom)        AS geom_json,
@@ -60,8 +73,9 @@ export async function GET(request: Request) {
             FROM nearby
           ) AS percentile_rank,
           (SELECT count(*)::int FROM nearby) AS nearby_count
-        FROM estate_value_zones ez, pt
+        FROM estate_value_zones ez, pt, latest_year
         WHERE ST_Within(pt.geom, ez.geom)
+          AND (ez.year IS NULL OR ez.year = latest_year.yr)
         LIMIT 1
       `),
 
@@ -77,10 +91,16 @@ export async function GET(request: Request) {
             WITH pt AS (
               SELECT ST_SetSRID(ST_Point(${lng}, ${lat}), 4326) AS geom
             ),
+            latest_year AS (
+              SELECT MAX(year) AS yr
+              FROM estate_value_zones
+              WHERE source = 'boris-ni' AND city_slug = ${cityParam}
+            ),
             clicked_zone AS (
-              SELECT value_eur_m2
+              SELECT ez.value_eur_m2
-              FROM estate_value_zones, pt
+              FROM estate_value_zones ez, pt, latest_year
-              WHERE ST_Within(pt.geom, estate_value_zones.geom)
+              WHERE ST_Within(pt.geom, ez.geom)
+                AND (ez.year IS NULL OR ez.year = latest_year.yr)
               LIMIT 1
             ),
             clicked_gp_score AS (
@@ -99,7 +119,7 @@ export async function GET(request: Request) {
             ),
             zone_scores AS (
               SELECT ez.value_eur_m2, nearest.composite_score
-              FROM estate_value_zones ez
+              FROM estate_value_zones ez, latest_year
               JOIN LATERAL (
                 SELECT AVG(gs.score) AS composite_score
                 FROM grid_scores gs
@@ -116,6 +136,7 @@ export async function GET(request: Request) {
               ) nearest ON true
               WHERE ez.value_eur_m2 IS NOT NULL
                 AND ez.city_slug = ${cityParam}
+                AND (ez.year IS NULL OR ez.year = latest_year.yr)
             ),
             peer_zones AS (
               SELECT value_eur_m2
@@ -159,7 +180,7 @@ export async function GET(request: Request) {
     );
   }
 
-  // ── Bbox query: ?bbox=minLng,minLat,maxLng,maxLat ─────────────────────────
+  // ── Bbox query: ?bbox=minLng,minLat,maxLng,maxLat&city=slug ───────────────
   const bboxParam = searchParams.get("bbox");
   if (!bboxParam) {
     return NextResponse.json({ error: "bbox or lat/lng required" }, { status: 400 });
@@ -171,6 +192,7 @@ export async function GET(request: Request) {
   }
 
   const [minLng, minLat, maxLng, maxLat] = parts;
+  const cityParam = searchParams.get("city");
 
   const rows = await Promise.resolve(sql<{
     geom_json: string;
@@ -182,14 +204,22 @@ export async function GET(request: Request) {
     stichtag: string | null;
   }[]>`
     SELECT
-      ST_AsGeoJSON(geom) AS geom_json,
+      ST_AsGeoJSON(ez.geom) AS geom_json,
-      value_eur_m2::float AS value,
+      ez.value_eur_m2::float AS value,
-      zone_name, usage_type, usage_detail, dev_state, stichtag
+      ez.zone_name, ez.usage_type, ez.usage_detail, ez.dev_state, ez.stichtag
-    FROM estate_value_zones
+    FROM estate_value_zones ez
-    WHERE ST_Intersects(
+    ${cityParam
-      geom,
+      ? sql`
-      ST_MakeEnvelope(${minLng}, ${minLat}, ${maxLng}, ${maxLat}, 4326)
+        JOIN LATERAL (
-    )
+          SELECT MAX(year) AS yr
+          FROM estate_value_zones
+          WHERE source = 'boris-ni' AND city_slug = ${cityParam}
+        ) latest_year ON (ez.year IS NULL OR ez.year = latest_year.yr)
+        WHERE ez.city_slug = ${cityParam}
+          AND ST_Intersects(ez.geom, ST_MakeEnvelope(${minLng}, ${minLat}, ${maxLng}, ${maxLat}, 4326))`
+      : sql`
+        WHERE ST_Intersects(ez.geom, ST_MakeEnvelope(${minLng}, ${minLat}, ${maxLng}, ${maxLat}, 4326))`
+    }
     LIMIT 5000
   `);
 

apps/web/app/page.tsx

@@ -181,12 +181,15 @@ export default function HomePage() {
     };
   }, [pinLocation, estateValueAvailable, selectedCity, mode, threshold, profile]);
 
-  // Fetch isochrone when in isochrone mode with an active pin.
+  // Pre-fetch the isochrone whenever a pin is placed (in accessibility mode).
-  // Isochrones are only shown in accessibility mode — switching to estate-value
+  // overlayMode is intentionally NOT a dep — fetching is decoupled from display.
-  // or hidden-gem clears the isochrone so those overlays become visible.
+  // This means:
-  // Switching back to accessibility re-fetches the isochrone automatically.
+  //   - Switching to "relative" doesn't discard the data.
+  //   - Switching back to "isochrone" shows the isochrone instantly (no re-fetch).
+  // Isochrone data is cleared when switching to a non-accessibility overlay
+  // (estate-value / hidden-gem), since those overlays replace the isochrone slot.
   useEffect(() => {
-    if (!pinLocation || overlayMode !== "isochrone" || baseOverlay !== "accessibility") {
+    if (!pinLocation || baseOverlay !== "accessibility") {
       setIsochroneData(null);
       return;
     }
@@ -221,7 +224,7 @@ export default function HomePage() {
       cancelled = true;
       setIsochroneLoading(false);
     };
-  }, [pinLocation, overlayMode, mode, threshold, baseOverlay]);
+  }, [pinLocation, mode, threshold, baseOverlay]); // eslint-disable-line react-hooks/exhaustive-deps
 
   function handleProfileChange(newProfile: ProfileId) {
     setProfile(newProfile);

apps/web/components/map-view.tsx

@@ -196,6 +196,14 @@ export function MapView({
     if (src?.setTiles) src.setTiles([tileUrl(citySlug, mode, threshold, profile)]);
   }, [mapLoaded, citySlug, mode, threshold, profile]);
 
+  // ── Zoom to city bbox when city changes ───────────────────────────────────
+  // Destructure to primitive deps so the effect only fires on actual value changes.
+  const [bboxW, bboxS, bboxE, bboxN] = cityBbox;
+  useEffect(() => {
+    if (!mapLoaded) return;
+    mapRef.current!.fitBounds([bboxW, bboxS, bboxE, bboxN], { padding: 40, duration: 800 });
+  }, [mapLoaded, bboxW, bboxS, bboxE, bboxN]);
+
   // ── Pin marker ─────────────────────────────────────────────────────────────
   // Markers are DOM elements (not style layers), but mapRef.current is only
   // set inside the mount effect which runs after all earlier effects. Adding
@@ -354,9 +362,10 @@ export function MapView({
     if (!showEstateValue) return;
 
     const [minLng, minLat, maxLng, maxLat] = stateRef.current.cityBbox;
+    const city = stateRef.current.citySlug;
     let cancelled = false;
 
-    fetch(`/api/estate-value?bbox=${minLng},${minLat},${maxLng},${maxLat}`)
+    fetch(`/api/estate-value?bbox=${minLng},${minLat},${maxLng},${maxLat}&city=${encodeURIComponent(city)}`)
       .then((r) => {
         if (!r.ok) throw new Error(`estate-value API ${r.status}`);
         return r.json();

infra/schema.sql

@@ -153,8 +153,15 @@ CREATE TABLE IF NOT EXISTS estate_value_zones (
   dev_state    TEXT,
   stichtag     TEXT,
   source       TEXT NOT NULL DEFAULT 'boris-ni',
+  year         SMALLINT,
   ingested_at  TIMESTAMPTZ NOT NULL DEFAULT NOW()
 );
 
+-- Migration for existing databases
+ALTER TABLE estate_value_zones ADD COLUMN IF NOT EXISTS year SMALLINT;
+
 CREATE INDEX IF NOT EXISTS idx_estate_value_zones_city ON estate_value_zones (city_slug);
 CREATE INDEX IF NOT EXISTS idx_estate_value_zones_geom ON estate_value_zones USING GIST (geom);
+CREATE INDEX IF NOT EXISTS idx_estate_value_zones_year
+  ON estate_value_zones (city_slug, year)
+  WHERE year IS NOT NULL;

worker/src/jobs/compute-scores.ts

@@ -308,9 +308,16 @@ export async function handleComputeScores(
   // Each grid point looks up the nearest zone's price, ranks it within its accessibility
   // decile, and stores hidden_gem_score = composite_accessibility × (1 − price_rank).
   const gemThreshold = thresholds.includes(15) ? 15 : thresholds[0];
+  // Count only the latest year's zones so historical rows don't skew the check.
   const [{ n }] = await Promise.resolve(sql<{ n: number }[]>`
-    SELECT count(*)::int AS n FROM estate_value_zones
+    SELECT count(*)::int AS n
-    WHERE city_slug = ${citySlug} AND value_eur_m2 IS NOT NULL
+    FROM estate_value_zones ez
+    WHERE ez.city_slug = ${citySlug}
+      AND ez.value_eur_m2 IS NOT NULL
+      AND (ez.year IS NULL OR ez.year = (
+        SELECT MAX(year) FROM estate_value_zones
+        WHERE city_slug = ${citySlug} AND source = 'boris-ni' AND year IS NOT NULL
+      ))
   `);
   if (n > 0) {
     await job.updateProgress({
@@ -319,16 +326,23 @@ export async function handleComputeScores(
       message: "Computing hidden gem scores…",
     } satisfies JobProgress);
     await Promise.resolve(sql`
-      WITH grid_with_price AS (
+      WITH latest_year AS (
-        -- For each grid point, get composite accessibility score and nearest zone price
+        SELECT MAX(year) AS yr
+        FROM estate_value_zones
+        WHERE city_slug = ${citySlug} AND source = 'boris-ni'
+      ),
+      grid_with_price AS (
+        -- For each grid point, get composite accessibility score and nearest latest-year zone price
         SELECT
           gp.id,
           COALESCE(AVG(gs.score), 0)                    AS composite_score,
           ROUND(COALESCE(AVG(gs.score), 0) * 10)::int   AS score_decile,
           (
             SELECT ez.value_eur_m2
-            FROM estate_value_zones ez
+            FROM estate_value_zones ez, latest_year
-            WHERE ez.city_slug = ${citySlug} AND ez.value_eur_m2 IS NOT NULL
+            WHERE ez.city_slug = ${citySlug}
+              AND ez.value_eur_m2 IS NOT NULL
+              AND (ez.year IS NULL OR ez.year = latest_year.yr)
             ORDER BY gp.geom <-> ez.geom
             LIMIT 1
           ) AS value_eur_m2

worker/src/jobs/ingest-boris-ni.ts

@@ -1,7 +1,11 @@
 /**
  * Ingest BORIS NI (Niedersachsen Bodenrichtwerte) estate value zones.
  *
- * Fetches from the LGLN WFS and stores into estate_value_zones.
+ * Fetches all available years from the LGLN WFS, starting from the current year
+ * and working backwards until a year returns an ExceptionReport (not yet published).
+ * All years are stored with a `year` column; analysis always uses the latest year.
+ *
+ * Historical URL pattern: boris_{year}_wfs (e.g. boris_2024_wfs)
  * Only enqueued by refresh-city when the city's bbox intersects Niedersachsen.
  */
 import type { Job } from "bullmq";
@@ -13,8 +17,9 @@ export type IngestBorisNiData = {
   citySlug: string;
 };
 
-const WFS_BASE = "https://opendata.lgln.niedersachsen.de/doorman/noauth/boris_wfs";
+const WFS_HOST = "https://opendata.lgln.niedersachsen.de/doorman/noauth";
 const CHUNK = 500;
+const EARLIEST_YEAR = 2011;
 
 // ─── GML Parser (WFS 2.0 / GML 3.2) ──────────────────────────────────────────
 
@@ -128,11 +133,83 @@ function parseGmlFeatures(gml: string): EstateValueFeature[] {
   return features;
 }
 
+function buildWfsUrl(endpoint: string, minlat: number, minlng: number, maxlat: number, maxlng: number): string {
+  const url = new URL(`${WFS_HOST}/${endpoint}`);
+  url.searchParams.set("SERVICE", "WFS");
+  url.searchParams.set("VERSION", "2.0.0");
+  url.searchParams.set("REQUEST", "GetFeature");
+  url.searchParams.set("TYPENAMES", "boris:BR_BodenrichtwertZonal");
+  url.searchParams.set("SRSNAME", "urn:ogc:def:crs:EPSG::4326");
+  url.searchParams.set("BBOX", `${minlat},${minlng},${maxlat},${maxlng},urn:ogc:def:crs:EPSG::4326`);
+  url.searchParams.set("COUNT", "10000");
+  return url.toString();
+}
+
+/**
+ * Fetches a WFS URL and returns the GML response text, or null if the year
+ * dataset doesn't exist (ExceptionReport, non-2xx, or network error).
+ */
+async function tryFetchWfs(url: string): Promise<string | null> {
+  let response: Response;
+  try {
+    response = await fetch(url, {
+      headers: { Accept: "application/gml+xml; version=3.2" },
+      signal: AbortSignal.timeout(60_000),
+    });
+  } catch {
+    return null;
+  }
+  if (!response.ok) return null;
+  const gml = await response.text();
+  if (gml.includes("<ows:ExceptionReport")) return null;
+  return gml;
+}
+
+async function insertFeatures(
+  sql: ReturnType<typeof getSql>,
+  citySlug: string,
+  year: number,
+  features: EstateValueFeature[],
+): Promise<void> {
+  for (let i = 0; i < features.length; i += CHUNK) {
+    const chunk = features.slice(i, i + CHUNK);
+    const geomJsons    = chunk.map((f) => JSON.stringify(f.geometry));
+    const values       = chunk.map((f) => f.value);
+    const zoneNames    = chunk.map((f) => f.zoneName);
+    const usageTypes   = chunk.map((f) => f.usageType);
+    const usageDetails = chunk.map((f) => f.usageDetail);
+    const devStates    = chunk.map((f) => f.devState);
+    const stichtags    = chunk.map((f) => f.stichtag);
+
+    await Promise.resolve(sql`
+      INSERT INTO estate_value_zones
+        (city_slug, geom, value_eur_m2, zone_name, usage_type, usage_detail, dev_state, stichtag, source, year)
+      SELECT
+        ${citySlug},
+        ST_SetSRID(ST_GeomFromGeoJSON(g), 4326),
+        v,
+        zn, ut, ud, ds, st,
+        'boris-ni',
+        ${year}::smallint
+      FROM unnest(
+        ${geomJsons}::text[],
+        ${values}::numeric[],
+        ${zoneNames}::text[],
+        ${usageTypes}::text[],
+        ${usageDetails}::text[],
+        ${devStates}::text[],
+        ${stichtags}::text[]
+      ) AS t(g, v, zn, ut, ud, ds, st)
+    `);
+  }
+}
+
 // ─── Job handler ──────────────────────────────────────────────────────────────
 
 export async function handleIngestBorisNi(job: Job<IngestBorisNiData>): Promise<void> {
   const { citySlug } = job.data;
   const sql = getSql();
+  const currentYear = new Date().getFullYear();
 
   await job.updateProgress({
     stage: "Ingesting BORIS NI",
@@ -157,106 +234,55 @@ export async function handleIngestBorisNi(job: Job<IngestBorisNiData>): Promise<
 
   const { minlng, minlat, maxlng, maxlat } = bboxRows[0];
 
-  await job.updateProgress({
+  // Clear all existing BORIS NI data for this city before re-ingesting all years.
-    stage: "Ingesting BORIS NI",
-    pct: 10,
-    message: `Fetching BORIS NI WFS for ${citySlug}`,
-  } satisfies JobProgress);
-
-  // Fetch from BORIS NI WFS (lat/lon axis order for EPSG:4326)
-  const wfsUrl = new URL(WFS_BASE);
-  wfsUrl.searchParams.set("SERVICE", "WFS");
-  wfsUrl.searchParams.set("VERSION", "2.0.0");
-  wfsUrl.searchParams.set("REQUEST", "GetFeature");
-  wfsUrl.searchParams.set("TYPENAMES", "boris:BR_BodenrichtwertZonal");
-  wfsUrl.searchParams.set("SRSNAME", "urn:ogc:def:crs:EPSG::4326");
-  wfsUrl.searchParams.set("BBOX", `${minlat},${minlng},${maxlat},${maxlng},urn:ogc:def:crs:EPSG::4326`);
-  wfsUrl.searchParams.set("COUNT", "10000");
-
-  const response = await fetch(wfsUrl.toString(), {
-    headers: { Accept: "application/gml+xml; version=3.2" },
-    signal: AbortSignal.timeout(60_000),
-  });
-
-  if (!response.ok) {
-    throw new Error(`BORIS NI WFS returned HTTP ${response.status}`);
-  }
-
-  const gml = await response.text();
-
-  if (gml.includes("<ows:ExceptionReport")) {
-    console.warn(`[ingest-boris-ni] WFS returned exception report for ${citySlug}`);
-    return;
-  }
-
-  await job.updateProgress({
-    stage: "Ingesting BORIS NI",
-    pct: 50,
-    message: "Parsing GML response",
-  } satisfies JobProgress);
-
-  const features = parseGmlFeatures(gml);
-  console.log(`[ingest-boris-ni] Parsed ${features.length} zones for ${citySlug}`);
-
-  // Replace all existing BORIS NI data for this city
   await Promise.resolve(sql`
     DELETE FROM estate_value_zones WHERE city_slug = ${citySlug} AND source = 'boris-ni'
   `);
 
-  if (features.length === 0) {
+  // Fetch from current year downwards. An ExceptionReport means the year dataset
+  // doesn't exist yet (not yet published by LGLN), so we stop there.
+  let totalInserted = 0;
+  let yearsProcessed = 0;
+
+  for (let year = currentYear; year >= EARLIEST_YEAR; year--) {
     await job.updateProgress({
       stage: "Ingesting BORIS NI",
-      pct: 100,
+      pct: 5 + Math.round((yearsProcessed / (currentYear - EARLIEST_YEAR + 1)) * 90),
-      message: `No zones found for ${citySlug}`,
+      message: `Fetching year ${year}…`,
     } satisfies JobProgress);
-    return;
+
+    // Try the year-specific endpoint first.
+    let gml = await tryFetchWfs(buildWfsUrl(`boris_${year}_wfs`, minlat, minlng, maxlat, maxlng));
+
+    // For the current year, also try the unversioned endpoint as a fallback —
+    // it points to the latest published data and may be available before the
+    // year-specific URL is set up.
+    if (gml === null && year === currentYear) {
+      console.log(`[ingest-boris-ni] Year ${year}: year URL unavailable, trying unversioned boris_wfs`);
+      gml = await tryFetchWfs(buildWfsUrl("boris_wfs", minlat, minlng, maxlat, maxlng));
+    }
+
+    if (gml === null) {
+      console.log(`[ingest-boris-ni] Year ${year}: no data available — stopping`);
+      break;
+    }
+
+    const features = parseGmlFeatures(gml);
+    console.log(`[ingest-boris-ni] Year ${year}: ${features.length} zones for ${citySlug}`);
+
+    if (features.length > 0) {
+      await insertFeatures(sql, citySlug, year, features);
+      totalInserted += features.length;
+    }
+
+    yearsProcessed++;
   }
 
-  let inserted = 0;
+  console.log(`[ingest-boris-ni] ✓ Stored ${totalInserted} BORIS NI zones across ${yearsProcessed} years for ${citySlug}`);
-  for (let i = 0; i < features.length; i += CHUNK) {
-    const chunk = features.slice(i, i + CHUNK);
-
-    const geomJsons    = chunk.map((f) => JSON.stringify(f.geometry));
-    const values       = chunk.map((f) => f.value);
-    const zoneNames    = chunk.map((f) => f.zoneName);
-    const usageTypes   = chunk.map((f) => f.usageType);
-    const usageDetails = chunk.map((f) => f.usageDetail);
-    const devStates    = chunk.map((f) => f.devState);
-    const stichtags    = chunk.map((f) => f.stichtag);
-
-    await Promise.resolve(sql`
-      INSERT INTO estate_value_zones
-        (city_slug, geom, value_eur_m2, zone_name, usage_type, usage_detail, dev_state, stichtag, source)
-      SELECT
-        ${citySlug},
-        ST_SetSRID(ST_GeomFromGeoJSON(g), 4326),
-        v,
-        zn, ut, ud, ds, st,
-        'boris-ni'
-      FROM unnest(
-        ${geomJsons}::text[],
-        ${values}::numeric[],
-        ${zoneNames}::text[],
-        ${usageTypes}::text[],
-        ${usageDetails}::text[],
-        ${devStates}::text[],
-        ${stichtags}::text[]
-      ) AS t(g, v, zn, ut, ud, ds, st)
-    `);
-
-    inserted += chunk.length;
-    await job.updateProgress({
-      stage: "Ingesting BORIS NI",
-      pct: 50 + Math.round((inserted / features.length) * 50),
-      message: `Stored ${inserted}/${features.length} zones`,
-    } satisfies JobProgress);
-  }
-
-  console.log(`[ingest-boris-ni] ✓ Stored ${inserted} BORIS NI zones for ${citySlug}`);
 
   await job.updateProgress({
     stage: "Ingesting BORIS NI",
     pct: 100,
-    message: `BORIS NI ingest complete: ${inserted} zones`,
+    message: `BORIS NI ingest complete: ${totalInserted} zones across ${yearsProcessed} years`,
   } satisfies JobProgress);
 }