diff --git a/.env.example b/.env.example index a81f2c4..bf2fbe5 100644 --- a/.env.example +++ b/.env.example @@ -129,6 +129,18 @@ AI_GENERATION_RETENTION_DAYS=180 # Import-Extraktion ab. 0 = unbegrenzt. AI_MONTHLY_TOKEN_LIMIT=0 +# --- Craftvia: Planung – Karten & Geocoding (L13) --- +# Adresse → Koordinaten nur serverseitig im Worker (Job geocode-site, max. 1 Anfrage/s), +# Ergebnis wird am Objekt gespeichert. nominatim | none (none = keine Verortung, Objekte +# erscheinen „ohne Ortsangabe“). Für Produktion eigenen/vertraglichen Dienst verwenden. +GEOCODING_PROVIDER=nominatim +# GEOCODING_URL=https://nominatim.openstreetmap.org +# Eindeutiger User-Agent (Nominatim-Richtlinie); leer = "Craftvia/ (+APP_BASE_URL)". +# GEOCODING_USER_AGENT= +# Kartenkacheln der Live-Lage; der Host wird beim Build in die CSP (img-src) übernommen. +# MAP_TILE_URL=https://tile.openstreetmap.org/{z}/{x}/{y}.png +# MAP_ATTRIBUTION=© OpenStreetMap-Mitwirkende + # --- Optionale Fundament-Variablen (Default leer) --- # MFA_ENC_KEY: Schlüssel für TOTP-Secrets at-rest (leer = aus AUTH_SECRET abgeleitet). # ⚠ Nach dem Setzen nicht mehr ändern. diff --git a/scripts/craftvia-worker.ts b/scripts/craftvia-worker.ts index 649e67e..f2a53de 100644 --- a/scripts/craftvia-worker.ts +++ b/scripts/craftvia-worker.ts @@ -18,8 +18,9 @@ async function main() { continue; } const processor = await load(); - const concurrency = name === JOB_QUEUES.reportPdf ? 2 : 4; - const w = new Worker(name, async (job) => processor(job.data), { connection, concurrency }); + const geocode = name === JOB_QUEUES.geocodeSite; // L13: OSM Nominatim policy — max. 1 request/s + const concurrency = name === JOB_QUEUES.reportPdf ? 2 : geocode ? 1 : 4; + const w = new Worker(name, async (job) => processor(job.data), { connection, concurrency, ...(geocode ? { limiter: { max: 1, duration: 1_000 } } : {}) }); w.on("failed", (job, err) => console.error(`[worker] ${name} job ${job?.id} failed:`, err.message)); workers.push(w); console.info(`[worker] listening on ${name}`); diff --git a/scripts/geocode-backfill.ts b/scripts/geocode-backfill.ts new file mode 100644 index 0000000..0d68ab6 --- /dev/null +++ b/scripts/geocode-backfill.ts @@ -0,0 +1,81 @@ +/** + * L13 Planung — geocode sites that have no (or outdated) coordinates, respecting the OSM Nominatim + * usage policy: strictly sequential, max. 1 request per second (process-wide limiter in the + * provider), identifying User-Agent, results cached on the site (unchanged addresses are skipped). + * Manually entered coordinates are never overwritten. + * + * Usage: + * npx tsx scripts/geocode-backfill.ts # all tenants + * npx tsx scripts/geocode-backfill.ts --tenant=demo --limit=50 + * npx tsx scripts/geocode-backfill.ts --dry-run # only list what would be geocoded + * npx tsx scripts/geocode-backfill.ts --retry-not-found # also retry addresses marked not_found + * + * For larger data sets use a self-hosted or contracted geocoding service (GEOCODING_URL). + */ +import "dotenv/config"; +import { isValidLatLng } from "../src/lib/geo/distance"; +import { dbForTenant, prisma } from "../src/server/db"; +import { geocodingConfig } from "../src/server/services/geo/config"; +import { geocodeSite, type GeocodeOutcome } from "../src/server/services/geo/geocode-site"; +import { geocodeQueryFor } from "../src/server/services/geo/normalize"; + +const arg = (name: string) => process.argv.find((a) => a.startsWith(`--${name}=`))?.split("=")[1]; +const flag = (name: string) => process.argv.includes(`--${name}`); + +async function main() { + const cfg = geocodingConfig(); + const dryRun = flag("dry-run"); + const retryNotFound = flag("retry-not-found"); + const limit = Number(arg("limit") ?? Infinity); + const slug = arg("tenant"); + if (cfg.provider === "none" && !dryRun) { + console.log("GEOCODING_PROVIDER=none — nichts zu tun."); + return; + } + console.log(`Provider ${cfg.provider} (${cfg.url}), User-Agent "${cfg.userAgent}", ≤ 1 Anfrage/s${dryRun ? " — Probelauf" : ""}`); + + const tenants = await prisma.tenant.findMany({ where: slug ? { slug } : {}, select: { id: true, slug: true }, orderBy: { slug: "asc" } }); + const totals: Record = {}; + let done = 0; + for (const tenant of tenants) { + const db = dbForTenant(tenant.id); + const sites = await db.site.findMany({ + where: { deletedAt: null }, + select: { id: true, name: true, street: true, houseNumber: true, postalCode: true, city: true, country: true, latitude: true, longitude: true, geocodedAt: true, geocodeStatus: true, geocodeQuery: true }, + orderBy: { createdAt: "asc" }, + }); + const pending = sites.filter((s) => { + const query = geocodeQueryFor(s); + if (!query) return false; + if (isValidLatLng(s) && !s.geocodedAt) return false; // manual coordinates + if (s.geocodeQuery !== query) return true; + if (s.geocodeStatus === "ok") return false; + if (s.geocodeStatus === "not_found") return retryNotFound; + return true; + }); + if (!pending.length) continue; + console.log(`\n${tenant.slug}: ${pending.length} Objekt(e)`); + for (const site of pending) { + if (done >= limit) break; + done++; + if (dryRun) { + console.log(` · ${site.name} (${[site.postalCode, site.city].filter(Boolean).join(" ")})`); + continue; + } + if (retryNotFound && site.geocodeStatus === "not_found") await db.site.update({ where: { id: site.id }, data: { geocodeStatus: null } }); + const outcome: GeocodeOutcome = await geocodeSite(tenant.id, site.id); + totals[outcome] = (totals[outcome] ?? 0) + 1; + console.log(` ${outcome.padEnd(9)} ${site.name}`); + } + } + console.log(`\nFertig: ${done} Objekt(e) ${JSON.stringify(totals)}`); +} + +main() + .catch((err) => { + console.error(err); + process.exitCode = 1; + }) + .finally(async () => { + await prisma.$disconnect(); + }); diff --git a/scripts/test-planung-geocode.ts b/scripts/test-planung-geocode.ts new file mode 100644 index 0000000..8f834dc --- /dev/null +++ b/scripts/test-planung-geocode.ts @@ -0,0 +1,178 @@ +// L13 Planung — Geocoding: Normalisierung/Änderungserkennung, Nominatim-Anfrage (URL, User-Agent, +// Accept-Language, Auswertung, Fehler) mit injiziertem fetch, Drosselung 1/s mit Fake-Uhr, Processor mit +// Fake-Provider (Cache bei unveränderter Adresse, not_found, failed → Retry, manuelle Koordinaten, +// Provider none, Mandantentrennung), Hook beim Anlegen ohne Inline-Aufruf. Nominatim wird NIE aufgerufen. +// Lauf: npx tsx scripts/test-planung-geocode.ts + +import "dotenv/config"; +import { prisma } from "../src/server/db"; +import { createTenant, ok, runSuite, section } from "./lib/e2e-fixture"; +import { geocodingConfig, mapConfig } from "../src/server/services/geo/config"; +import { requestSiteGeocoding } from "../src/server/services/geo/dispatch"; +import { geocodeSite } from "../src/server/services/geo/geocode-site"; +import { buildNominatimUrl, createNominatimProvider } from "../src/server/services/geo/nominatim"; +import { addressChanged, geocodeQueryFor } from "../src/server/services/geo/normalize"; +import { GeocodingError, setGeocodingProvider, type GeocodeAddress, type GeocodingProvider } from "../src/server/services/geo/provider"; +import { createIntervalLimiter } from "../src/server/services/geo/rate-limit"; +import { process as geocodeProcessor } from "../src/server/jobs/processors/geocode-site"; +import { PROCESSORS } from "../src/server/jobs/processors"; +import { createSite } from "../src/server/services/sites/sites"; + +const SLUG_A = "zz-planung-geo-a"; +const SLUG_B = "zz-planung-geo-b"; + +// Guard: any request to a Nominatim host fails the suite. +let nominatimHits = 0; +const realFetch = globalThis.fetch; +globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input instanceof Request ? input.url : String(input); + if (/nominatim|openstreetmap/i.test(url)) { + nominatimHits++; + throw new Error("Nominatim must never be called from tests"); + } + return realFetch(input, init); +}) as typeof fetch; + +function fakeProvider(answer: (a: GeocodeAddress) => { status: "ok"; latitude: number; longitude: number } | { status: "not_found" } | "throw") { + const calls: GeocodeAddress[] = []; + const provider: GeocodingProvider = { + name: "fake", + async geocode(a) { + calls.push(a); + const r = answer(a); + if (r === "throw") throw new GeocodingError("http 503"); + return r; + }, + }; + return { provider, calls }; +} + +runSuite("L13 Planung – Geocoding", [SLUG_A, SLUG_B], async () => { + section("Normalisierung & Änderungserkennung"); + const addr = { street: "Hafenstraße", houseNumber: "12", postalCode: "20457", city: "Hamburg", country: "DE" }; + ok(geocodeQueryFor(addr) === "hafenstraße 12|20457|hamburg|de", `normierte Adresse (${geocodeQueryFor(addr)})`); + ok(geocodeQueryFor({ ...addr, street: " HAFENSTRASSE ".replace("SS", "ß"), city: "hamburg " }) === geocodeQueryFor(addr), "Groß-/Kleinschreibung und Leerzeichen egal"); + ok(geocodeQueryFor({ street: "Hafenstraße", houseNumber: "12" }) === null, "ohne PLZ und Ort → nicht verortbar"); + ok(!addressChanged(addr, { ...addr, city: "HAMBURG" }) && addressChanged(addr, { ...addr, houseNumber: "14" }), "nur relevante Adressänderungen zählen"); + + section("Nominatim-Anfrage (ohne Netz)"); + const url = buildNominatimUrl("https://nominatim.example/", addr); + ok( + url.pathname === "/search" && url.searchParams.get("format") === "jsonv2" && url.searchParams.get("limit") === "1" && url.searchParams.get("street") === "12 Hafenstraße" && url.searchParams.get("postalcode") === "20457" && url.searchParams.get("city") === "Hamburg" && url.searchParams.get("countrycodes") === "de", + `strukturierte Suche (${url.search})`, + ); + const cfg = geocodingConfig({ APP_BASE_URL: "https://app.craftvia.example" }); + ok(/^Craftvia\/\d+\.\d+\.\d+ \(\+https:\/\/app\.craftvia\.example\)$/.test(cfg.userAgent) && cfg.url === "https://nominatim.openstreetmap.org" && cfg.provider === "nominatim", `Standard-User-Agent „${cfg.userAgent}“`); + ok(geocodingConfig({ GEOCODING_USER_AGENT: "Betrieb/1 (+mailto:it@example.org)", GEOCODING_PROVIDER: "none", GEOCODING_URL: "https://geo.example/" }).userAgent === "Betrieb/1 (+mailto:it@example.org)", "User-Agent per GEOCODING_USER_AGENT überschreibbar"); + ok(geocodingConfig({ GEOCODING_PROVIDER: "none" }).provider === "none" && geocodingConfig({ GEOCODING_URL: "https://geo.example/" }).url === "https://geo.example", "Provider none / eigene GEOCODING_URL"); + ok(mapConfig({}).tileUrl === "https://tile.openstreetmap.org/{z}/{x}/{y}.png" && mapConfig({}).attribution.includes("OpenStreetMap"), "Kartenkacheln: OSM-Standard mit Namensnennung"); + + const seen: { url: string; headers: Headers }[] = []; + let reply: () => Response = () => Response.json([{ lat: "53.5413", lon: "9.9849" }]); + const acquired: number[] = []; + const nominatim = createNominatimProvider( + { url: "https://nominatim.example", userAgent: "Craftvia/test (+https://app.example)", timeoutMs: 1000 }, + { + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + seen.push({ url: String(input), headers: new Headers(init?.headers) }); + return reply(); + }) as typeof fetch, + limiter: { acquire: async () => (acquired.push(Date.now()), Date.now()) }, + }, + ); + const hit = await nominatim.geocode(addr); + ok(hit.status === "ok" && hit.latitude === 53.5413 && hit.longitude === 9.9849, "Treffer ausgewertet"); + ok(seen[0]?.headers.get("user-agent") === "Craftvia/test (+https://app.example)" && seen[0].headers.get("accept-language") === "de", "User-Agent und Accept-Language de gesetzt"); + ok(acquired.length === 1, "jede Anfrage läuft über den Drosselungs-Limiter"); + reply = () => Response.json([]); + ok((await nominatim.geocode(addr)).status === "not_found", "leere Antwort → not_found"); + reply = () => new Response("busy", { status: 503 }); + let threw = false; + try { + await nominatim.geocode(addr); + } catch (err) { + threw = err instanceof GeocodingError; + } + ok(threw, "HTTP-Fehler → GeocodingError (wiederholbar)"); + + section("Drosselung 1 Anfrage/s (Fake-Uhr)"); + let clock = 10_000; + const sleeps: number[] = []; + const limiter = createIntervalLimiter(1000, { now: () => clock, sleep: async (ms) => void sleeps.push(ms) }); + const slots = [await limiter.acquire(), await limiter.acquire(), await limiter.acquire()]; + ok(slots[0] === 10_000 && slots[1] === 11_000 && slots[2] === 12_000 && sleeps.join(",") === "1000,2000", `Slots im Sekundenabstand (${slots.join(",")}), Wartezeiten ${sleeps.join(",")}`); + clock = 20_000; + ok((await limiter.acquire()) === 20_000 && sleeps.length === 2, "nach Pause sofort wieder frei"); + + section("geocodeSite mit Fake-Provider"); + const A = await createTenant(SLUG_A); + const B = await createTenant(SLUG_B); + const fake = fakeProvider((a) => (a.city === "Nirgendwo" ? { status: "not_found" } : { status: "ok", latitude: 53.55, longitude: a.houseNumber === "14" ? 10.01 : 10.0 })); + setGeocodingProvider(fake.provider); + const site = await prisma.site.create({ data: { tenantId: A.tenantId, customerId: A.customerId, name: "Geo 1", ...addr } }); + + ok((await geocodeSite(A.tenantId, site.id)) === "ok" && fake.calls.length === 1, "neue Adresse → ok, 1 Provider-Aufruf"); + const s1 = await prisma.site.findUniqueOrThrow({ where: { id: site.id } }); + ok(s1.latitude === 53.55 && s1.longitude === 10.0 && s1.geocodeStatus === "ok" && s1.geocodeQuery === geocodeQueryFor(addr) && !!s1.geocodedAt, "Koordinaten, Status, Adresse und Zeitpunkt am Objekt gespeichert"); + ok((await prisma.auditLog.count({ where: { tenantId: A.tenantId, entity: "site", entityId: site.id, action: "update" } })) === 1, "Koordinatenänderung auditiert"); + ok((await geocodeSite(A.tenantId, site.id)) === "cached" && fake.calls.length === 1, "unveränderte Adresse → Cache, kein weiterer Aufruf"); + await prisma.site.update({ where: { id: site.id }, data: { city: "hamburg " } }); + ok((await geocodeSite(A.tenantId, site.id)) === "cached" && fake.calls.length === 1, "nur Schreibweise geändert → Cache"); + await prisma.site.update({ where: { id: site.id }, data: { houseNumber: "14" } }); + ok((await geocodeSite(A.tenantId, site.id)) === "ok" && fake.calls.length === 2 && (await prisma.site.findUniqueOrThrow({ where: { id: site.id } })).longitude === 10.01, "geänderte Adresse → neu verortet"); + await prisma.site.update({ where: { id: site.id }, data: { city: "Nirgendwo", postalCode: "99999" } }); + ok((await geocodeSite(A.tenantId, site.id)) === "not_found" && fake.calls.length === 3, "unbekannte Adresse → not_found"); + const s2 = await prisma.site.findUniqueOrThrow({ where: { id: site.id } }); + ok(s2.geocodeStatus === "not_found" && s2.latitude === null && s2.longitude === null, "not_found entfernt veraltete automatische Koordinaten"); + ok((await geocodeSite(A.tenantId, site.id)) === "cached" && fake.calls.length === 3, "not_found wird für dieselbe Adresse nicht wiederholt"); + + const failing = fakeProvider(() => "throw"); + const siteFail = await prisma.site.create({ data: { tenantId: A.tenantId, customerId: A.customerId, name: "Geo Fehler", ...addr } }); + ok((await geocodeSite(A.tenantId, siteFail.id, { provider: failing.provider })) === "failed" && (await prisma.site.findUniqueOrThrow({ where: { id: siteFail.id } })).geocodeStatus === "failed", "Providerfehler → Status failed"); + ok((await geocodeSite(A.tenantId, siteFail.id)) === "ok", "failed wird beim nächsten Lauf erneut versucht"); + + const manual = await prisma.site.create({ data: { tenantId: A.tenantId, customerId: A.customerId, name: "Geo manuell", ...addr, latitude: 50.1, longitude: 8.6 } }); + const callsBefore = fake.calls.length; + ok((await geocodeSite(A.tenantId, manual.id)) === "manual" && fake.calls.length === callsBefore, "manuelle Koordinaten → kein Aufruf"); + const m = await prisma.site.findUniqueOrThrow({ where: { id: manual.id } }); + ok(m.latitude === 50.1 && m.longitude === 8.6 && m.geocodeStatus === "skipped", "manuelle Koordinaten bleiben unverändert (Status skipped)"); + + const none = await prisma.site.create({ data: { tenantId: A.tenantId, customerId: A.customerId, name: "Geo none", ...addr } }); + ok((await geocodeSite(A.tenantId, none.id, { provider: null })) === "skipped" && (await prisma.site.findUniqueOrThrow({ where: { id: none.id } })).latitude === null, "Provider none → skipped, ohne Ortsangabe, kein Fehler"); + const noAddr = await prisma.site.create({ data: { tenantId: A.tenantId, customerId: A.customerId, name: "Geo ohne Adresse", street: "Irgendwo" } }); + ok((await geocodeSite(A.tenantId, noAddr.id)) === "skipped" && fake.calls.length === callsBefore, "unvollständige Adresse → skipped ohne Aufruf"); + + const bCalls = fake.calls.length; + ok((await geocodeSite(B.tenantId, none.id)) === "missing" && fake.calls.length === bCalls, "Mandant B: Objekt von A nicht gefunden, kein Aufruf"); + ok((await prisma.site.findUniqueOrThrow({ where: { id: none.id } })).geocodeStatus === "skipped", "Objekt von A unverändert"); + + section("Processor & Hook"); + ok(typeof PROCESSORS["geocode-site"] === "function", "Processor geocode-site registriert"); + const viaJob = await prisma.site.create({ data: { tenantId: A.tenantId, customerId: A.customerId, name: "Geo Job", ...addr } }); + await geocodeProcessor({ tenantId: A.tenantId, entityId: viaJob.id }); + ok((await prisma.site.findUniqueOrThrow({ where: { id: viaJob.id } })).geocodeStatus === "ok", "Processor verortet das Objekt"); + setGeocodingProvider(failing.provider); + const viaJobFail = await prisma.site.create({ data: { tenantId: A.tenantId, customerId: A.customerId, name: "Geo Job Fehler", ...addr } }); + let jobThrew = false; + try { + await geocodeProcessor({ tenantId: A.tenantId, entityId: viaJobFail.id }); + } catch { + jobThrew = true; + } + ok(jobThrew, "Processor wirft bei failed → BullMQ wiederholt"); + setGeocodingProvider(fake.provider); + + const prev = process.env.GEOCODING_PROVIDER; + process.env.GEOCODING_PROVIDER = "none"; + ok((await requestSiteGeocoding(A.tenantId, viaJob.id)) === false, "GEOCODING_PROVIDER=none → kein Job"); + process.env.GEOCODING_PROVIDER = "nominatim"; + const hookCalls = fake.calls.length; + const created = await createSite(A.ctx.backoffice, { customerId: A.customerId, name: "Geo Hook", street: "Am Kaiserkai", houseNumber: "1", postalCode: "20457", city: "Hamburg" }); + await new Promise((resolve) => setTimeout(resolve, 100)); + ok(!!created.id && fake.calls.length === hookCalls, "Anlage eines Objekts: Verortung nur per Queue, nie inline im Request"); + if (prev === undefined) delete process.env.GEOCODING_PROVIDER; + else process.env.GEOCODING_PROVIDER = prev; + setGeocodingProvider(undefined); + + ok(nominatimHits === 0, "Nominatim wurde in keinem Test aufgerufen"); +}); diff --git a/src/server/jobs/processors/geocode-site.ts b/src/server/jobs/processors/geocode-site.ts new file mode 100644 index 0000000..a114c33 --- /dev/null +++ b/src/server/jobs/processors/geocode-site.ts @@ -0,0 +1,12 @@ +import { geocodeSite } from "@/server/services/geo/geocode-site"; +import type { JobPayload } from "../queues"; + +/** + * Queue "geocode-site" (L13 Planung): address → coordinates for one site, cached on the row. + * The worker runs this queue with concurrency 1 + BullMQ limiter 1/s; the Nominatim provider + * additionally throttles process-wide. `failed` is rethrown so BullMQ retries with backoff. + */ +export async function process(payload: JobPayload): Promise { + const outcome = await geocodeSite(payload.tenantId, payload.entityId); + if (outcome === "failed") throw new Error("geocoding failed (retry scheduled)"); +} diff --git a/src/server/services/geo/config.ts b/src/server/services/geo/config.ts new file mode 100644 index 0000000..f5031d0 --- /dev/null +++ b/src/server/services/geo/config.ts @@ -0,0 +1,42 @@ +import packageJson from "../../../../package.json"; + +/** + * Geocoding + map configuration (L13 Planung). Defaults follow the OpenStreetMap usage policies: + * Nominatim only server side, max. 1 request/s, identifying User-Agent, results cached per site. + * For production with more than a handful of addresses a self-hosted / contracted geocoding and + * tile service is recommended (see docs/craftvia/lanes/planung.md). + */ + +export const GEOCODING_PROVIDERS = ["nominatim", "none"] as const; +export type GeocodingProviderName = (typeof GEOCODING_PROVIDERS)[number]; + +export const DEFAULT_NOMINATIM_URL = "https://nominatim.openstreetmap.org"; +export const DEFAULT_MAP_TILE_URL = "https://tile.openstreetmap.org/{z}/{x}/{y}.png"; +export const DEFAULT_MAP_ATTRIBUTION = "© OpenStreetMap-Mitwirkende"; + +type Env = Record; +const val = (v: string | undefined) => (v && v.trim() ? v.trim() : undefined); + +export function geocodingProviderName(env: Env = process.env): GeocodingProviderName { + const v = val(env.GEOCODING_PROVIDER)?.toLowerCase(); + return v === "none" ? "none" : "nominatim"; +} + +export function geocodingConfig(env: Env = process.env) { + const baseUrl = val(env.APP_BASE_URL) ?? val(env.AUTH_URL) ?? "http://localhost:3000"; + return { + provider: geocodingProviderName(env), + url: (val(env.GEOCODING_URL) ?? DEFAULT_NOMINATIM_URL).replace(/\/+$/, ""), + userAgent: val(env.GEOCODING_USER_AGENT) ?? `Craftvia/${packageJson.version} (+${baseUrl})`, + timeoutMs: 8_000, + minIntervalMs: 1_000, + }; +} + +/** Tile URL + attribution for the live map (passed from server pages to the client). */ +export function mapConfig(env: Env = process.env) { + return { + tileUrl: val(env.MAP_TILE_URL) ?? DEFAULT_MAP_TILE_URL, + attribution: val(env.MAP_ATTRIBUTION) ?? DEFAULT_MAP_ATTRIBUTION, + }; +} diff --git a/src/server/services/geo/dispatch.ts b/src/server/services/geo/dispatch.ts new file mode 100644 index 0000000..642885f --- /dev/null +++ b/src/server/services/geo/dispatch.ts @@ -0,0 +1,24 @@ +import { enqueueJob } from "@/server/jobs/queues"; +import { geocodingProviderName } from "@/server/services/geo/config"; + +/** + * Ask the worker to geocode a site (after create / address change / import confirmation). + * Never runs inline: no external request inside a user request (OSM policy: no bulk geocoding in + * requests). If the queue is not reachable yet (lazy Redis connection) one delayed retry is made; + * otherwise the site stays without coordinates ("ohne Ortsangabe") until the next change or + * `scripts/geocode-backfill.ts`. Never throws. + */ +export async function requestSiteGeocoding(tenantId: string, siteId: string): Promise { + if (geocodingProviderName() === "none") return false; + const payload = { tenantId, entityId: siteId }; + try { + if (await enqueueJob("geocode-site", payload)) return true; + } catch (err) { + console.error("[geo] enqueue geocode-site failed:", (err as Error).message); + } + const timer = setTimeout(() => { + enqueueJob("geocode-site", payload).catch(() => undefined); + }, 1_500); + timer.unref?.(); + return false; +} diff --git a/src/server/services/geo/geocode-site.ts b/src/server/services/geo/geocode-site.ts new file mode 100644 index 0000000..b088e26 --- /dev/null +++ b/src/server/services/geo/geocode-site.ts @@ -0,0 +1,92 @@ +import { isValidLatLng } from "@/lib/geo/distance"; +import { writeAuditLog } from "@/server/audit"; +import { dbForTenant } from "@/server/db"; +import { geocodeQueryFor } from "@/server/services/geo/normalize"; +import { getGeocodingProvider, type GeocodingProvider } from "@/server/services/geo/provider"; + +/** + * Geocode one site and cache the result on the row (job `geocode-site`, backfill script). + * + * Rules: + * - Manually entered coordinates (coordinates without `geocodedAt`) are never overwritten. + * - Unchanged address (`geocodeQuery`) with status ok/not_found → no provider call ("cached"). + * - No usable address or provider `none` → status `skipped` (retried once enabled/complete). + * - Transport/HTTP error → status `failed` (the job rethrows so BullMQ retries). + * - Address changed and not found → previous automatic coordinates are removed (they would be wrong). + */ + +export type GeocodeOutcome = "ok" | "not_found" | "failed" | "skipped" | "cached" | "manual" | "missing"; + +const SITE_SELECT = { + id: true, + street: true, + houseNumber: true, + postalCode: true, + city: true, + country: true, + latitude: true, + longitude: true, + geocodedAt: true, + geocodeStatus: true, + geocodeQuery: true, +} as const; + +export async function geocodeSite( + tenantId: string, + siteId: string, + opts: { provider?: GeocodingProvider | null; now?: Date } = {}, +): Promise { + const db = dbForTenant(tenantId); + const site = await db.site.findFirst({ where: { id: siteId, deletedAt: null }, select: SITE_SELECT }); + if (!site) return "missing"; + const now = opts.now ?? new Date(); + const query = geocodeQueryFor(site); + + if (isValidLatLng(site) && !site.geocodedAt) { + if (site.geocodeStatus !== "skipped" || site.geocodeQuery !== query) { + await db.site.update({ where: { id: site.id }, data: { geocodeStatus: "skipped", geocodeQuery: query } }); + } + return "manual"; + } + + if (!query) { + const clearAuto = site.geocodedAt !== null && (site.latitude !== null || site.longitude !== null); + await db.site.update({ + where: { id: site.id }, + data: { geocodeStatus: "skipped", geocodeQuery: null, ...(clearAuto ? { latitude: null, longitude: null } : {}) }, + }); + return "skipped"; + } + + if (site.geocodeQuery === query && (site.geocodeStatus === "ok" || site.geocodeStatus === "not_found")) return "cached"; + + const provider = opts.provider === undefined ? await getGeocodingProvider() : opts.provider; + if (!provider) { + if (site.geocodeStatus !== "skipped") await db.site.update({ where: { id: site.id }, data: { geocodeStatus: "skipped" } }); + return "skipped"; + } + + let result; + try { + result = await provider.geocode({ street: site.street, houseNumber: site.houseNumber, postalCode: site.postalCode, city: site.city, country: site.country }); + } catch (err) { + console.error(`[geo] geocoding site ${site.id} failed:`, (err as Error).message); + await db.site.update({ where: { id: site.id }, data: { geocodeStatus: "failed", geocodeQuery: query, geocodedAt: now } }); + return "failed"; + } + + const before = { latitude: site.latitude, longitude: site.longitude, geocodeStatus: site.geocodeStatus }; + const coords = result.status === "ok" ? { latitude: result.latitude, longitude: result.longitude } : { latitude: null, longitude: null }; + await db.site.update({ where: { id: site.id }, data: { ...coords, geocodeStatus: result.status, geocodeQuery: query, geocodedAt: now } }); + if (before.latitude !== coords.latitude || before.longitude !== coords.longitude) { + await writeAuditLog({ + tenantId, + action: "update", + entity: "site", + entityId: site.id, + before, + after: { ...coords, geocodeStatus: result.status, source: "geocoding", provider: provider.name }, + }); + } + return result.status; +} diff --git a/src/server/services/geo/nominatim.ts b/src/server/services/geo/nominatim.ts new file mode 100644 index 0000000..d1f3070 --- /dev/null +++ b/src/server/services/geo/nominatim.ts @@ -0,0 +1,58 @@ +import { GeocodingError, type GeocodeAddress, type GeocodeResult, type GeocodingProvider } from "@/server/services/geo/provider"; +import { nominatimLimiter } from "@/server/services/geo/rate-limit"; + +/** + * Nominatim (OpenStreetMap) provider. Usage policy: server side only, max. 1 request/s process-wide, + * identifying User-Agent, results are cached in Site.latitude/longitude by the caller, no bulk + * geocoding inside requests (only the geocode-site job and scripts/geocode-backfill.ts call this). + */ + +type Config = { url: string; userAgent: string; timeoutMs: number }; +type Deps = { fetch: typeof fetch; limiter: { acquire(): Promise } }; + +/** Structured search URL (no free text → fewer ambiguous hits). */ +export function buildNominatimUrl(baseUrl: string, address: GeocodeAddress): URL { + const url = new URL(`${baseUrl.replace(/\/+$/, "")}/search`); + url.searchParams.set("format", "jsonv2"); + url.searchParams.set("limit", "1"); + url.searchParams.set("addressdetails", "0"); + const street = [address.houseNumber, address.street].map((s) => s?.trim()).filter(Boolean).join(" "); + if (street) url.searchParams.set("street", street); + if (address.postalCode?.trim()) url.searchParams.set("postalcode", address.postalCode.trim()); + if (address.city?.trim()) url.searchParams.set("city", address.city.trim()); + const country = (address.country ?? "DE").trim().toLowerCase(); + if (/^[a-z]{2}$/.test(country)) url.searchParams.set("countrycodes", country); + return url; +} + +export function createNominatimProvider(config: Config, deps: Deps = { fetch: globalThis.fetch, limiter: nominatimLimiter }): GeocodingProvider { + return { + name: "nominatim", + async geocode(address: GeocodeAddress): Promise { + await deps.limiter.acquire(); + let res: Response; + try { + res = await deps.fetch(buildNominatimUrl(config.url, address), { + headers: { "User-Agent": config.userAgent, "Accept-Language": "de", Accept: "application/json" }, + signal: AbortSignal.timeout(config.timeoutMs), + }); + } catch (err) { + throw new GeocodingError(`request failed: ${(err as Error).name}`); + } + if (!res.ok) throw new GeocodingError(`http ${res.status}`); + let body: unknown; + try { + body = await res.json(); + } catch { + throw new GeocodingError("invalid response"); + } + const hit = Array.isArray(body) ? (body[0] as { lat?: unknown; lon?: unknown } | undefined) : undefined; + const latitude = Number(hit?.lat); + const longitude = Number(hit?.lon); + if (!hit || !Number.isFinite(latitude) || !Number.isFinite(longitude) || Math.abs(latitude) > 90 || Math.abs(longitude) > 180) { + return { status: "not_found" }; + } + return { status: "ok", latitude, longitude }; + }, + }; +} diff --git a/src/server/services/geo/normalize.ts b/src/server/services/geo/normalize.ts new file mode 100644 index 0000000..0081fef --- /dev/null +++ b/src/server/services/geo/normalize.ts @@ -0,0 +1,27 @@ +/** + * Normalized address string of a site ("hafenstraße 12|20457|hamburg|de"). Stored in + * Site.geocodeQuery next to the coordinates: an unchanged address is never geocoded twice. + * Returns null when the address is too incomplete to geocode (neither postal code nor city). + */ +export type SiteAddress = { + street?: string | null; + houseNumber?: string | null; + postalCode?: string | null; + city?: string | null; + country?: string | null; +}; + +const clean = (v: string | null | undefined) => (v ?? "").normalize("NFC").trim().replace(/\s+/g, " ").toLowerCase(); + +export function geocodeQueryFor(site: SiteAddress): string | null { + const postal = clean(site.postalCode); + const city = clean(site.city); + if (!postal && !city) return null; + const street = [clean(site.street), clean(site.houseNumber)].filter(Boolean).join(" "); + return [street, postal, city, clean(site.country) || "de"].join("|"); +} + +/** True when the geocoding-relevant part of the address changed. */ +export function addressChanged(before: SiteAddress, after: SiteAddress): boolean { + return geocodeQueryFor(before) !== geocodeQueryFor(after); +} diff --git a/src/server/services/geo/provider.ts b/src/server/services/geo/provider.ts new file mode 100644 index 0000000..ebe95fa --- /dev/null +++ b/src/server/services/geo/provider.ts @@ -0,0 +1,41 @@ +import { geocodingConfig, geocodingProviderName } from "@/server/services/geo/config"; + +/** Provider-neutral geocoding contract (L13 Planung) — swap Nominatim for another service via env. */ + +export type GeocodeAddress = { + street: string | null; + houseNumber: string | null; + postalCode: string | null; + city: string | null; + country: string | null; +}; + +export type GeocodeResult = { status: "ok"; latitude: number; longitude: number } | { status: "not_found" }; + +export interface GeocodingProvider { + readonly name: string; + /** Resolves ok/not_found; throws GeocodingError for transport/HTTP failures (retryable). */ + geocode(address: GeocodeAddress): Promise; +} + +export class GeocodingError extends Error { + constructor(message: string) { + super(message); + this.name = "GeocodingError"; + } +} + +let override: GeocodingProvider | null | undefined; + +/** Tests inject a fake provider (never Nominatim); `undefined` restores the env-based provider. */ +export function setGeocodingProvider(provider: GeocodingProvider | null | undefined): void { + override = provider; +} + +/** Configured provider, or null when geocoding is disabled (`GEOCODING_PROVIDER=none`). */ +export async function getGeocodingProvider(): Promise { + if (override !== undefined) return override; + if (geocodingProviderName() === "none") return null; + const { createNominatimProvider } = await import("@/server/services/geo/nominatim"); + return createNominatimProvider(geocodingConfig()); +} diff --git a/src/server/services/geo/rate-limit.ts b/src/server/services/geo/rate-limit.ts new file mode 100644 index 0000000..e857ba8 --- /dev/null +++ b/src/server/services/geo/rate-limit.ts @@ -0,0 +1,24 @@ +/** + * Minimum-interval limiter: callers are released one after another with at least `intervalMs` + * between two slots (Nominatim policy: max. 1 request per second). Clock is injectable for tests. + */ +export type Clock = { now: () => number; sleep: (ms: number) => Promise }; + +const realClock: Clock = { now: () => Date.now(), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) }; + +export function createIntervalLimiter(intervalMs: number, clock: Clock = realClock) { + let next = 0; + return { + /** Waits for the next free slot and returns its timestamp. */ + async acquire(): Promise { + const now = clock.now(); + const slot = Math.max(now, next); + next = slot + intervalMs; + if (slot > now) await clock.sleep(slot - now); + return slot; + }, + }; +} + +/** Process-wide limiter for all Nominatim requests of this process (worker, backfill script). */ +export const nominatimLimiter = createIntervalLimiter(1_000); diff --git a/src/server/services/imports/confirm.ts b/src/server/services/imports/confirm.ts index 9104cd2..0ffab91 100644 --- a/src/server/services/imports/confirm.ts +++ b/src/server/services/imports/confirm.ts @@ -8,6 +8,7 @@ import { computeCorrections, reviewFormSchema, type ReviewForm } from "@/lib/imp import { createWorkOrder } from "@/server/services/work-orders/create"; import { startExtraction, type Dispatch } from "./upload"; import { dispatchJob } from "@/server/jobs/dispatch"; +import { requestSiteGeocoding } from "@/server/services/geo/dispatch"; export type ConfirmResult = { workOrderId: string; workOrderNumber: string; customerId: string; siteId: string | null; contactId: string | null }; @@ -168,6 +169,7 @@ export async function confirmImport(ctx: ServiceCtx, importId: string, rawForm: if (result.createdCustomer) await writeAuditLog({ ...base, action: "create", entity: "customer", entityId: result.customerId, after: { source: "import", importId: job.id } }); if (result.contactId) await writeAuditLog({ ...base, action: "create", entity: "contact", entityId: result.contactId, after: { customerId: result.customerId, source: "import" } }); if (result.createdSite && result.siteId) await writeAuditLog({ ...base, action: "create", entity: "site", entityId: result.siteId, after: { customerId: result.customerId, source: "import" } }); + if (result.createdSite && result.siteId) void requestSiteGeocoding(ctx.tenantId, result.siteId); // L13 Planung: queued geocoding of the new site // work order creation is audited by services/work-orders/create (entity work_order) await writeAuditLog({ ...base, diff --git a/src/server/services/sites/sites.ts b/src/server/services/sites/sites.ts index c0380d4..dc1ebb0 100644 --- a/src/server/services/sites/sites.ts +++ b/src/server/services/sites/sites.ts @@ -4,6 +4,8 @@ import { writeAuditLog } from "@/server/audit"; import { assertCan, ServiceError, type ServiceCtx } from "@/server/services/context"; import { siteScope } from "@/server/services/work-orders/visibility"; import { optStr } from "@/server/services/customers/schemas"; +import { requestSiteGeocoding } from "@/server/services/geo/dispatch"; +import { addressChanged } from "@/server/services/geo/normalize"; export const SITE_STATUSES = ["active", "inactive", "provisional"] as const; @@ -121,6 +123,7 @@ export async function createSite(ctx: ServiceCtx, input: SiteCreateInput) { await assertCustomerAndContact(ctx, data.customerId, data.contactId); const site = await ctx.db.site.create({ data: { ...data, tenantId: ctx.tenantId, country: data.country ?? "DE" } }); await writeAuditLog({ tenantId: ctx.tenantId, actorId: ctx.userId, action: "create", entity: "site", entityId: site.id, after: site }); + void requestSiteGeocoding(ctx.tenantId, site.id); // L13 Planung: queued, never inline return site; } @@ -134,6 +137,7 @@ export async function updateSite(ctx: ServiceCtx, id: string, patch: SitePatchIn await assertCustomerAndContact(ctx, customerId, contactId); const after = await ctx.db.site.update({ where: { id }, data: { ...data, contactId } }); await writeAuditLog({ tenantId: ctx.tenantId, actorId: ctx.userId, action: "update", entity: "site", entityId: id, before, after }); + if (addressChanged(before, after)) void requestSiteGeocoding(ctx.tenantId, id); // L13 Planung return after; }