cache derived geometry: merged road chains + settlement list persisted with cell fingerprints; two-tier best-first road stitching (raw-node headings, permissive interchange pass); fast path skips all raw parsing on unchanged data

This commit is contained in:
Richard Zink committed 2026-10-10 13:15:03 +02:00
1 parent 99b29f216a
commit aebe311e32
2 files changed
+370 -29

No files matched your search

+1 -2
View File
@@ -30,8 +30,7 @@ real terrain, driven by the real sun, moon, stars, and live weather.
mood (sunny / cloudy / stormy / night).
- **Fully cached, offline-capable**: basemap tiles, elevation grid,
Overpass road/place data, star catalogue and the last weather payload
live permanently in IndexedDB. Once warm, the scene renders fully
offline; a configurable expiry (`CONFIG.tiles.expireDays`) can refresh
live permanently in IndexedDB. Once warm, the scene renders fully offline, and a 15-minute cycle transparently retries anything that previously failed. Merged road chains and the decoded settlement list are cached as well and only recomputed when an Overpass cell is actually re-fetched.tiles.expireDays`) can refresh
basemap tiles periodically.
- **Location-aware**: set `CONFIG.location.lat/lon`; everything
(flight anchor, terrain tiles, sun/moon, weather, star rotation) is
+369 -27
View File
@@ -1977,13 +1977,21 @@ function resamplePoly(pts) {
leftover += seg;
prev = p;
}
// keep the ORIGINAL end point: OSM ways of one road share exact
// junction coordinates, and carMergeChains matches those seams
var end = pts[pts.length - 1];
var last = out[out.length - 1];
if (Math.hypot(end.x - last.x, end.z - last.z) > 1e-9) {
out.push({ x: end.x, z: end.z, d: last.d + leftover });
}
return out;
}
function fetchMotorways() {
if (!CAR_ON || carFetching) return;
carFetching = true;
var fresh = 0; // cells that were NOT already cached this cycle
var fresh = 0; // cells that needed a real fetch this cycle
var fpParts = []; // "id:savedAt" of every raw cell -> fp()
var groups = { motorway: [], secondary: [] };
function parseWays(txt) {
var data;
@@ -1998,7 +2006,35 @@ function fetchMotorways() {
return { x: carLonToX(g.lon), z: carLatToZ(g.lat) };
});
var rp = resamplePoly(pts);
if (rp.length > 1) groups[key].push(rp);
if (rp.length > 1) {
// local heading at both ends from RAW consecutive nodes
// (resampled 2 km chords cut curves too aggressively for
// seam-direction matching on real motorway curves)
var h0 = pts[1], h1 = pts[0];
var t0 = pts[pts.length - 1], t1 = pts[pts.length - 2];
var hl = Math.hypot(h0.x - h1.x, h0.z - h1.z) || 1;
var tl = Math.hypot(t0.x - t1.x, t0.z - t1.z) || 1;
rp.headD = { x: (h1.x - h0.x) / hl, z: (h1.z - h0.z) / hl }; // away from body
rp.tailD = { x: (t0.x - t1.x) / tl, z: (t0.z - t1.z) / tl };
groups[key].push(rp);
}
});
}
// stitch the raw ways into continuous chains, rebuild ribbons and
// (optionally on cycle completion) persist the chains for fast loads
function rebuildChains(persist) {
var mCh = carMergeChains(groups.motorway);
var bCh = carMergeChains(groups.secondary);
carRawCounts = [groups.motorway.length, groups.secondary.length];
carGeo = { motorway: mCh, secondary: bCh };
buildCarwayVBO(elevGl || (clouds3dState && clouds3dState.gl));
if (!persist) return;
var em = encodeChains(mCh), eb = encodeChains(bCh);
openTileDB().then(function (db) {
if (!db) return;
tileDbPut(db, { id: 'geo/roadchains_v1', fp: cellsFp(fpParts),
mCnts: em.cnts, mData: em.data,
bCnts: eb.cnts, bData: eb.data, savedAt: Date.now() });
});
}
// rebuild ribbons shortly after each arriving cell - the lights grow
@@ -2008,8 +2044,7 @@ function fetchMotorways() {
if (rebuildT) return;
rebuildT = setTimeout(function () {
rebuildT = null;
carGeo = groups;
buildCarwayVBO(elevGl || (clouds3dState && clouds3dState.gl));
rebuildChains(false);
}, 1500);
}
function loadCell(entry) {
@@ -2017,36 +2052,65 @@ function fetchMotorways() {
return tileDbGet(db, entry.id).then(function (rec) {
// validate stored JSON - a poisoned cache record (HTML error
// page saved by an older build) must trigger a refetch
if (rec && rec.json && isValidOverpass(rec.json)) { entry._cached = true; return rec.json; }
if (rec && rec.json && isValidOverpass(rec.json)) {
entry._cached = true;
fpParts.push(entry.id + ':' + rec.savedAt);
return rec.json;
}
return opTry(entry.q, entry.start).then(function (txt) {
if (txt && db) tileDbPut(db, { id: entry.id, json: txt, savedAt: Date.now() });
if (txt && db) {
var ts = Date.now();
fpParts.push(entry.id + ':' + ts);
tileDbPut(db, { id: entry.id, json: txt, savedAt: ts });
}
return txt;
});
});
}).then(function (txt) {
if (txt) {
parseWays(txt);
// cached cells are re-parsed but never trigger a rebuild;
// only freshly fetched data changes the ribbons
if (!entry._cached) { fresh++; scheduleRebuild(); }
if (!entry._cached) fresh++;
if (!entry._cached || !carGeo) scheduleRebuild();
}
else console.warn('[clouds3d] overpass cell failed: ' + entry.id);
});
}
// cached cells resolve instantly; fresh ones are paced by the global
// gate. Failed cells are retried on the next 15-min cycle
// (successful ones stay cached forever).
// Fast path: merged chains are cached AND the fingerprint over all raw
// cell records is unchanged -> restore chains straight to the GPU.
// Fingerprint check reads record metadata only, no JSON parsing.
openTileDB().then(function (db) {
if (!db) return false;
return tileDbGet(db, 'geo/roadchains_v1').then(function (rec) {
if (!rec || !rec.mCnts || !rec.fp) return false;
var ok = true, parts = [];
return runPool(CAR_QUERIES, 8, function (e) {
if (!ok) return Promise.resolve();
return tileDbGet(db, e.id).then(function (r) {
if (!r || !isValidOverpass(r.json)) ok = false;
else parts.push(e.id + ':' + r.savedAt);
});
}).then(function () {
if (!ok || cellsFp(parts) !== rec.fp) return false;
carGeo = { motorway: decodeChains(rec.mCnts, rec.mData),
secondary: decodeChains(rec.bCnts, rec.bData) };
buildCarwayVBO(elevGl || (clouds3dState && clouds3dState.gl));
console.log('[clouds3d] car layer: ' + carGeo.motorway.length +
' motorway + ' + carGeo.secondary.length + ' chains (from cache)');
return true;
});
});
}).then(function (hit) {
if (hit) { carFetching = false; return; }
// slow path: fetch what is missing, then stitch + persist once
runPool(CAR_QUERIES, 2, function (entry) {
return loadCell(entry);
}).then(function () {
carFetching = false;
// a fully cached cycle with no prior failure means the ribbons
// are already in the state this cycle would rebuild - skip it
if (!fresh && carGeo) return;
carGeo = groups;
buildCarwayVBO(elevGl || (clouds3dState && clouds3dState.gl));
rebuildChains(true);
console.log('[clouds3d] car layer: ' + groups.motorway.length +
' motorway + ' + groups.secondary.length + ' secondary polylines');
' motorway + ' + groups.secondary.length + ' secondary polylines -> longest chain ~' +
Math.round(carMaxChainKm) + ' km');
});
});
}
@@ -2074,6 +2138,7 @@ function fetchSettlements() {
if (!CITY_ON || cityFetching) return;
cityFetching = true;
var fresh = 0;
var fpParts = [];
// fine 0.55 x 0.65 deg cells: every query answers in <20 s, which is
// the only reliable regime on the public mirrors (big boxes server-
// time out at any timeout: value and come back as unparseable junk).
@@ -2140,32 +2205,66 @@ function fetchSettlements() {
function loadCell(entry) {
return openTileDB().then(function (db) {
return tileDbGet(db, entry.id).then(function (rec) {
// validate stored JSON - a poisoned cache record (HTML error
// page saved by an older build) must trigger a refetch
if (rec && rec.json && isValidOverpass(rec.json)) { entry._cached = true; return rec.json; }
if (rec && rec.json && isValidOverpass(rec.json)) {
entry._cached = true;
fpParts.push(entry.id + ':' + rec.savedAt);
return rec.json;
}
return opTry(entry.q, entry.start).then(function (txt) {
if (txt && db) tileDbPut(db, { id: entry.id, json: txt, savedAt: Date.now() });
if (txt && db) {
var ts = Date.now();
fpParts.push(entry.id + ':' + ts);
tileDbPut(db, { id: entry.id, json: txt, savedAt: ts });
}
return txt;
});
});
}).then(function (txt) {
if (txt) {
parse(txt);
if (!entry._cached) { fresh++; scheduleRebuild(); }
if (!entry._cached) fresh++;
if (!entry._cached || !cityGeo) scheduleRebuild();
}
else console.warn('[clouds3d] place cell failed: ' + entry.id);
});
}
// fast path: decoded settlement list cached with a matching cell fp
openTileDB().then(function (db) {
if (!db) return false;
return tileDbGet(db, 'geo/placecache_v1').then(function (rec) {
if (!rec || !rec.cnt || !rec.fp) return false;
var ok = true, parts = [];
return runPool(cells, 8, function (e) {
if (!ok) return Promise.resolve();
return tileDbGet(db, e.id).then(function (r) {
if (!r || !isValidOverpass(r.json)) ok = false;
else parts.push(e.id + ':' + r.savedAt);
});
}).then(function () {
if (!ok || cellsFp(parts) !== rec.fp) return false;
cityGeo = decodePlaces(rec.cnt, rec.data);
buildCityVBO(elevGl || (clouds3dState && clouds3dState.gl));
console.log('[clouds3d] settlements: ' + cityGeo.length + ' places (from cache)');
return true;
});
});
}).then(function (hit) {
if (hit) { cityFetching = false; return; }
runPool(cells, 2, function (entry) {
return loadCell(entry);
}).then(function () {
cityFetching = false;
if (!fresh && cityGeo) return; // fully cached cycle - nothing changed
cityGeo = list.slice();
buildCityVBO(elevGl || (clouds3dState && clouds3dState.gl));
openTileDB().then(function (db) {
if (!db) return;
tileDbPut(db, { id: 'geo/placecache_v1', fp: cellsFp(fpParts),
cnt: list.length, data: encodePlaces(list), savedAt: Date.now() });
});
console.log('[clouds3d] settlements: ' + list.length + ' places -> ' + cityCount + ' lights' +
(cityCount ? '' : ' (terrain pending)'));
});
});
}
var CAR_VERT_SRC = [
@@ -2226,6 +2325,10 @@ var CAR_FRAG_SRC = [
// cars are slotted pulses; hash1(s) jitters each slot's position
// (varying spacing) and a per-slot sine wobble makes every vehicle
// crawl or overtake a bit (varying speeds). No trails - points only.
// Each slot also has a per-vehicle LIFE CYCLE: it switches on with
// a 5 s fade-in, drives for a random 1-20 min, fades out and rests
// for a random ~0.5-30 min before its next trip (on/off ramps in
// spirit) - the traffic field never sits on a fixed pattern.
'float flow = vLen / uDash + dir * uTime * uSpeed;',
// pulse width grows with distance (~1.5 px floor at the fade edge)
// so distant cars read as slightly larger, steady lights instead
@@ -2237,10 +2340,17 @@ var CAR_FRAG_SRC = [
'float s = n + float(k);',
'float vis = step(hash1(s + 17.3), dens);',
'if (vis > 0.0) {',
'float life = 60.0 + 1140.0 * hash1(s * 0.371 + 9.4);',
'float rest = 30.0 + 1770.0 * hash1(s * 0.617 + 3.2);',
'float per = life + rest;',
'float age = mod(uTime + 3607.0 * hash1(s * 0.913 + 7.7), per);',
'float on = smoothstep(0.0, 5.0, age) * (1.0 - smoothstep(life - 5.0, life, age));',
'if (on > 0.001) {',
'float off = 0.05 + hash1(s) * 0.8;',
'float w = 0.25 * sin(uTime * uSpeed * (1.5 + hash1(s + 7.1) * 4.0) + hash1(s + 3.7) * 6.283);',
'float d = abs(flow - (s + off + w));',
'inten = max(inten, max(0.0, 1.0 - d / slot));',
'inten = max(inten, on * max(0.0, 1.0 - d / slot));',
'}',
'}',
'}',
// beyond ~55 km merge the pulses into a steady traffic glow:
@@ -2299,6 +2409,170 @@ function motorwayDistField() {
return f;
}
// Chain OSM road segments into continuous routes by GLOBAL best-first
// stitching. Junctions are found PROXIMITY-based (not exact-node): many
// German motorways end each carriageway at its own node a few metres
// apart (bridges!), so exact-key matching severed chains at every
// overpass. A candidate join exists when two way-ends lie within ~130 m
// along the heading AND within ~32 m laterally (that lateral test is
// what keeps the two carriageways of a dual road from joining each
// other). Candidates are scored by directional continuity and processed
// best-first, so a mainline continuation always claims a junction before
// any ramp or angled fork can. Chains cap at ~580 km. Result: the A1
// Munich -> Vienna reads as ONE ribbon a light can traverse end to end.
var carMaxChainKm = 0;
var carRawCounts = [0, 0];
var carJoinCount = 0;
function carMergeChains(polys) {
var JOINED = 0;
var MAX_D = 45; // world units ~ 580 km
var NEAR = 0.03; // junction search radius (~390 m)
// two stitching tiers: TIGHT seams (same carriageway - bridge splits,
// shared nodes) claim junctions first; only still-open ends get the
// LOOSE pass (big interchange nodes where the mainline is cut ~150-300
// m apart with a small lateral jog). Lateral tolerance is what keeps
// parallel opposite carriageways from pairing with each other.
var TIERS = [
{ dot: 0.8, lat: 0.0012, alongLo: -0.0015, alongHi: 0.012 },
{ dot: 0.92, lat: 0.0035, alongLo: -0.004, alongHi: 0.026 }
];
var i, j, k;
var n = polys.length;
var wayLen = new Float64Array(n);
// endpoint records: position + unit vector pointing AWAY from the way
var eps = [];
var grid = {};
var cell = function (x, z) { return Math.floor(x / NEAR) + '_' + Math.floor(z / NEAR); };
for (i = 0; i < n; i++) {
var P = polys[i];
if (!P || P.length < 2) continue;
wayLen[i] = Math.max(P[0].d, P[P.length - 1].d);
var a = P[0], b = P[P.length - 1];
var ha = P.headD, ta = P.tailD;
if (!ha || !ta) {
var a2 = P[1], b2 = P[P.length - 2];
var al = Math.hypot(a.x - a2.x, a.z - a2.z) || 1;
var bl = Math.hypot(b.x - b2.x, b.z - b2.z) || 1;
ha = { x: (a.x - a2.x) / al, z: (a.z - a2.z) / al };
ta = { x: (b.x - b2.x) / bl, z: (b.z - b2.z) / bl };
}
var recs = [
{ w: i, e: 0, x: a.x, z: a.z, ax: ha.x, az: ha.z },
{ w: i, e: 1, x: b.x, z: b.z, ax: ta.x, az: ta.z }
];
for (k = 0; k < 2; k++) {
var r = recs[k];
var ck = cell(r.x, r.z);
(grid[ck] || (grid[ck] = [])).push(r);
eps.push(r);
}
}
// proximity candidate joins, per tier
var pairGeom = {};
function geomFor(A, B) {
var pk = A.w < B.w ? A.w + ':' + A.e + '|' + B.w + ':' + B.e
: B.w + ':' + B.e + '|' + A.w + ':' + A.e;
if (pairGeom[pk]) return pairGeom[pk];
var score = (-A.ax) * B.ax + (-A.az) * B.az;
var dirx = -A.ax, dirz = -A.az; // travel into junction
var gpx = B.x - A.x, gpz = B.z - A.z; // gap A.end -> B.start
var along = gpx * dirx + gpz * dirz;
var latx = gpx - along * dirx, latz = gpz - along * dirz;
var g = { s: score - Math.hypot(gpx, gpz) * 5, dot: score,
along: along, lat: Math.hypot(latx, latz) };
pairGeom[pk] = g;
return g;
}
var candsByTier = [[], []];
for (var t = 0; t < 2; t++) {
var T = TIERS[t];
var cands = candsByTier[t];
var seenPair = {};
for (i = 0; i < eps.length; i++) {
var A2 = eps[i];
var gx = Math.floor(A2.x / NEAR), gz = Math.floor(A2.z / NEAR);
for (var ox = -1; ox <= 1; ox++) {
for (var oz = -1; oz <= 1; oz++) {
var bucket = grid[(gx + ox) + '_' + (gz + oz)];
if (!bucket) continue;
for (j = 0; j < bucket.length; j++) {
var B2 = bucket[j];
if (B2.w === A2.w) continue;
var pk2 = A2.w < B2.w ? A2.w + ':' + A2.e + '|' + B2.w + ':' + B2.e
: B2.w + ':' + B2.e + '|' + A2.w + ':' + A2.e;
if (seenPair[pk2]) continue;
seenPair[pk2] = 1;
var gm = geomFor(A2, B2);
if (gm.dot <= T.dot) continue;
if (gm.lat > T.lat) continue; // opposite carriageway: reject
if (gm.along < T.alongLo || gm.along > T.alongHi) continue;
cands.push({ a: A2, b: B2, s: gm.s });
}
}
}
}
cands.sort(function (p, q) { return q.s - p.s; });
}
// union-find over ways with open-end bookkeeping
var parent = new Int32Array(n);
for (i = 0; i < n; i++) parent[i] = i;
function find(v) { while (parent[v] !== v) { parent[v] = parent[parent[v]]; v = parent[v]; } return v; }
var open = new Array(n), link = new Array(n), chainLen = new Float64Array(n);
for (i = 0; i < n; i++) { open[i] = [1, 1]; link[i] = [null, null]; chainLen[i] = wayLen[i]; }
for (var tc = 0; tc < 2; tc++) {
var candsT = candsByTier[tc];
for (var c = 0; c < candsT.length; c++) {
var ca = candsT[c].a, cb = candsT[c].b;
if (!open[ca.w][ca.e] || !open[cb.w][cb.e]) continue;
var ra = find(ca.w), rb = find(cb.w);
if (ra === rb) continue; // would close a loop
if (chainLen[ra] + chainLen[rb] > MAX_D) continue;
link[ca.w][ca.e] = { w: cb.w, e: cb.e };
link[cb.w][cb.e] = { w: ca.w, e: ca.e };
open[ca.w][ca.e] = 0;
open[cb.w][cb.e] = 0;
parent[rb] = ra;
chainLen[ra] += chainLen[rb];
JOINED++;
}
}
// walk each component from a free end, accumulating arc length
var out = [], started = new Uint8Array(n), maxD = 0;
for (i = 0; i < n; i++) {
if (started[i] || !polys[i] || polys[i].length < 2) continue;
var startSide = open[i][0] ? 0 : (open[i][1] ? 1 : -1);
if (startSide < 0) continue;
var chain = [], last = null, lastD = 0, cur = i, enter = startSide, guard = 0;
while (cur !== null && guard++ < 5000) {
started[cur] = 1;
var Q = polys[cur];
var s0 = enter === 1 ? Q.length - 1 : 0;
var st = enter === 1 ? -1 : 1;
for (var q = s0; q >= 0 && q < Q.length; q += st) {
var pt = Q[q];
var d = last === null ? pt.d : lastD + Math.hypot(pt.x - last.x, pt.z - last.z);
chain.push({ x: pt.x, z: pt.z, d: d });
last = pt; lastD = d;
}
var exit = enter === 0 ? 1 : 0;
var nxt = link[cur][exit];
if (nxt) { cur = nxt.w; enter = nxt.e; }
else cur = null;
}
if (chain.length > 1) {
var span = Math.max(chain[0].d, chain[chain.length - 1].d);
if (span > maxD) maxD = span;
out.push(chain);
}
}
// isolated ways (no joins at all) still form a chain of one
for (i = 0; i < n; i++) {
if (!started[i] && polys[i] && polys[i].length > 1) out.push(polys[i]);
}
if (n) { carMaxChainKm = maxD * KM_PER_TILE; carJoinCount = JOINED; } // empty input must not reset the previous call's stats
return out;
}
function buildCarwayVBO(gl) {
if (!CAR_ON || !gl || !carGeo || !terrainSample) return;
var ts = terrainSample;
@@ -2373,10 +2647,17 @@ function buildCarwayVBO(gl) {
gl.bindBuffer(gl.ARRAY_BUFFER, safe);
}
var m = upload(carVboM, build(carGeo.motorway, false));
// carGeo already carries MERGED chains: fetchMotorways stitches the
// raw ways once per data change and caches the chains (geo/roadchains)
var chainsM = carGeo.motorway;
var chainsB = carGeo.secondary;
var m = upload(carVboM, build(chainsM, false));
carVboM = m.vbo; carCountM = m.count;
var bn = upload(carVboB, build(carGeo.secondary, true));
var bn = upload(carVboB, build(chainsB, true));
carVboB = bn.vbo; carCountB = bn.count;
window.__carDbg = { motorwayIn: carRawCounts[0], secondaryIn: carRawCounts[1],
motorwayChains: chainsM.length, secondaryChains: chainsB.length,
maxChainKm: Math.round(carMaxChainKm * 10) / 10 };
sanitizeAttribSlots();
if (!carProg && (carCountM || carCountB)) {
@@ -2461,6 +2742,67 @@ function runPool(entries, lanes, worker) {
});
}
// ---- Cacheable derived geometry -----------------------------------------
// Raw Overpass JSON already lives in IndexedDB forever; the expensive
// post-processing (resample + chain stitching) is PERSISTED too and only
// redone when a cell is actually (re)fetched. A cellsFp() fingerprint over
// "id:savedAt" of every source record detects that; the fingerprint is
// built from record metadata only - no JSON parsing needed to check it.
function cellsFp(parts) {
var s = parts.slice().sort().join('|');
var h = 2166136261;
for (var i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 16777619); }
return (h >>> 0) + ':' + parts.length;
}
function u8ToB64(u8) {
var s = '';
for (var i = 0; i < u8.length; i += 8192)
s += String.fromCharCode.apply(null, u8.subarray(i, Math.min(i + 8192, u8.length)));
return btoa(s);
}
function b64ToU8(b64) {
var s = atob(b64), u = new Uint8Array(s.length);
for (var i = 0; i < s.length; i++) u[i] = s.charCodeAt(i);
return u;
}
function f32ToB64(f) { return u8ToB64(new Uint8Array(f.buffer, f.byteOffset, f.byteLength)); }
function b64ToF32(b64) { return new Float32Array(b64ToU8(b64).buffer); }
function encodeChains(chains) {
var cnts = [], total = 0, i, j;
for (i = 0; i < chains.length; i++) { cnts.push(chains[i].length); total += chains[i].length; }
var f = new Float32Array(total * 3), k = 0;
for (i = 0; i < chains.length; i++)
for (j = 0; j < chains[i].length; j++) {
var p = chains[i][j];
f[k++] = p.x; f[k++] = p.z; f[k++] = p.d;
}
return { cnts: cnts, data: f32ToB64(f) };
}
function decodeChains(cnts, b64) {
var f = b64ToF32(b64), out = [], k = 0;
for (var i = 0; i < cnts.length; i++) {
var arr = [];
for (var j = 0; j < cnts[i]; j++) { arr.push({ x: f[k], z: f[k + 1], d: f[k + 2] }); k += 3; }
out.push(arr);
}
return out;
}
function encodePlaces(list) {
var f = new Float32Array(list.length * 4);
for (var i = 0; i < list.length; i++) {
f[i * 4] = list[i].x; f[i * 4 + 1] = list[i].z;
f[i * 4 + 2] = list[i].rank; f[i * 4 + 3] = list[i].pop;
}
return f32ToB64(f);
}
function decodePlaces(cnt, b64) {
var f = b64ToF32(b64), out = [];
for (var i = 0; i < cnt; i++)
out.push({ x: f[i * 4], z: f[i * 4 + 1], rank: Math.round(f[i * 4 + 2]), pop: f[i * 4 + 3] });
return out;
}
// same guard as the car rebuild: after deleting the old city VBO every
// still-enabled attrib slot must point at a live buffer, or the next
// full-screen draw warns "no buffer is bound to enabled attribute".