zink/animated_wallpaper
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

geo: cell manifest geo/cells_v1 - cached road/place fast paths check one small record instead of reading every multi-megabyte raw cell

Browse Source
This commit is contained in:
Richard Zink committed 2026-10-11 21:34:52 +02:00
1 parent 2d78c7081c
commit efd4db47cb
1 file changed
+83 -44
Download Patch File Download Diff File Expand all files Collapse all files
vendor/clouds3d.js
Vendored
+83 -44
View File
@@ -2206,11 +2206,42 @@ function parseOverpass(t) {
return (d && Array.isArray(d.elements)) ? d : null;
} catch (e) { return null; }
}
// Fingerprint checks only need "is this record usable at all" - a full
// JSON.parse of every cached cell just to hash its metadata is what made
// even a warm cache cycle expensive.
function cheapOverpass(t) {
return !!t && t.length > 64 && t.indexOf('"elements"') >= 0;
// Cell manifest: one tiny record that maps every cached Overpass cell to the
// time it was stored. The cached fast paths used to read every raw cell
// (hundreds of multi-megabyte JSON blobs) just to prove the chain cache is
// still current - the manifest answers the same question in one small read.
// Records are only ever written after parseOverpass() accepted them, so a
// poisoned cell cannot enter the manifest.
var cellManifest = null;
var _manifestPutAt = 0;
function loadManifest(db) {
return tileDbGet(db, 'geo/cells_v1').then(function (rec) {
if (!cellManifest) cellManifest = {};
// merge, never replace: notes taken this session may not be on disk
// yet (writes are throttled) and must not be lost.
if (rec && rec.map) {
for (var k in rec.map) {
if (!cellManifest[k] || cellManifest[k] < rec.map[k]) cellManifest[k] = rec.map[k];
}
}
return cellManifest;
});
}
function manifestNote(id, savedAt) {
if (!cellManifest) cellManifest = {};
if (cellManifest[id] !== savedAt) {
cellManifest[id] = savedAt;
manifestPersist(false);
}
}
function manifestPersist(force) {
if (!cellManifest) return;
var now = Date.now();
if (!force && now - _manifestPutAt < 3000) return;
_manifestPutAt = now;
openTileDB().then(function (db) {
if (db) tileDbPut(db, { id: 'geo/cells_v1', map: cellManifest, savedAt: now });
});
}
function _opPump() {
var now = Date.now();
@@ -2503,6 +2534,7 @@ function fetchMotorways() {
if (cached) {
entry._cached = true;
fpParts.push(entry.id + ':' + rec.savedAt);
manifestNote(entry.id, rec.savedAt);
progCell(true);
return cached;
}
@@ -2513,6 +2545,7 @@ function fetchMotorways() {
if (db) {
var ts = Date.now();
fpParts.push(entry.id + ':' + ts);
manifestNote(entry.id, ts);
tileDbPut(db, { id: entry.id, json: txt, savedAt: ts });
progCell(false);
}
@@ -2537,6 +2570,7 @@ function fetchMotorways() {
// worker for geometry. No stitching and no raw-JSON parsing at all.
openTileDB().then(function (db) {
if (!db) return false;
return loadManifest(db).then(function (map) {
return tileDbGet(db, 'geo/roadchains_v1').then(function (rec) {
if (!rec || !rec.mCnts || !rec.ids) return false;
// Valid as long as the chains were built from the CURRENT query
@@ -2546,31 +2580,32 @@ function fetchMotorways() {
return rec.ids.indexOf(e.id) >= 0;
});
if (!sameSet) return false;
// The manifest lists the cached cells and their timestamps, so
// currency is checked without touching a single raw cell.
var newest = 0, missing = 0, total = CAR_QUERIES.length;
return runPool(CAR_QUERIES, 8, function (e) {
return tileDbGet(db, e.id).then(function (r) {
if (r && cheapOverpass(r.json)) { if (r.savedAt > newest) newest = r.savedAt; }
else missing++;
});
}).then(function () {
// Reuse only when the raw set is essentially complete. A
// partial build must stay rebuildable, otherwise a network
// that was assembled while the mirrors were banning us would
// be replayed forever and never grow.
if (newest > rec.savedAt) return false;
if (missing > Math.max(1, total * 0.1)) {
dbgLog('[clouds3d] car layer: ' + missing + '/' + total +
' cells missing - rebuilding instead of replaying the cache');
return false;
}
carGeo = { motorway: { cnts: Int32Array.from(rec.mCnts), pts: b64ToF32(rec.mData) },
secondary: { cnts: Int32Array.from(rec.bCnts), pts: b64ToF32(rec.bData) } };
kickCarBuild(false);
dbgLog('[clouds3d] car layer: ' + carGeo.motorway.cnts.length +
' motorway + ' + carGeo.secondary.cnts.length + ' chains (from cache' +
(missing ? ', ' + missing + ' cells unavailable' : '') + ')');
return true;
});
for (var i = 0; i < CAR_QUERIES.length; i++) {
var at = map[CAR_QUERIES[i].id];
if (!at) missing++;
else if (at > newest) newest = at;
}
// Reuse only when the raw set is essentially complete. A
// partial build must stay rebuildable, otherwise a network
// that was assembled while the mirrors were banning us would
// be replayed forever and never grow.
if (newest > rec.savedAt) return false;
if (missing > Math.max(1, total * 0.1)) {
dbgLog('[clouds3d] car layer: ' + missing + '/' + total +
' cells missing - rebuilding instead of replaying the cache');
return false;
}
carGeo = { motorway: { cnts: Int32Array.from(rec.mCnts), pts: b64ToF32(rec.mData) },
secondary: { cnts: Int32Array.from(rec.bCnts), pts: b64ToF32(rec.bData) } };
kickCarBuild(false);
dbgLog('[clouds3d] car layer: ' + carGeo.motorway.cnts.length +
' motorway + ' + carGeo.secondary.cnts.length + ' chains (from cache' +
(missing ? ', ' + missing + ' cells unavailable' : '') + ')');
return true;
});
});
}).then(function (hit) {
if (hit) { carFetching = false; return; }
@@ -2579,6 +2614,7 @@ function fetchMotorways() {
return loadCell(entry);
}).then(function () {
carFetching = false;
manifestPersist(true);
carPersistFp = geoFp(fpParts);
carCycleFp = carPersistFp;
kickCarBuild(true);
@@ -2691,6 +2727,7 @@ function fetchSettlements() {
if (cached) {
entry._cached = true;
fpParts.push(entry.id + ':' + rec.savedAt);
manifestNote(entry.id, rec.savedAt);
progCell(true);
return cached;
}
@@ -2701,6 +2738,7 @@ function fetchSettlements() {
if (db) {
var ts = Date.now();
fpParts.push(entry.id + ':' + ts);
manifestNote(entry.id, ts);
tileDbPut(db, { id: entry.id, json: txt, savedAt: ts });
progCell(false);
}
@@ -2725,23 +2763,22 @@ function fetchSettlements() {
// fast path: decoded settlement list cached with a matching cell fp
openTileDB().then(function (db) {
if (!db) return false;
return loadManifest(db).then(function (map) {
return tileDbGet(db, 'geo/placecache_v1').then(function (rec) {
if (!rec || !rec.cnt || !rec.fp) return false;
var ok = true, parts = [];
return runPool(cells, 8, function (e) {
if (!ok) return Promise.resolve();
return tileDbGet(db, e.id).then(function (r) {
if (!r || !cheapOverpass(r.json)) ok = false;
else parts.push(e.id + ':' + r.savedAt);
});
}).then(function () {
if (!ok || geoFp(parts) !== rec.fp) return false;
cityGeo = b64ToF32(rec.data);
kickCityBuild();
if (carGeo) kickCarBuild(false);
dbgLog('[clouds3d] settlements: ' + rec.cnt + ' places (from cache)');
return true;
});
var parts = [];
for (var i = 0; i < cells.length; i++) {
var at = map[cells[i].id];
if (!at) return false;
parts.push(cells[i].id + ':' + at);
}
if (geoFp(parts) !== rec.fp) return false;
cityGeo = b64ToF32(rec.data);
kickCityBuild();
if (carGeo) kickCarBuild(false);
dbgLog('[clouds3d] settlements: ' + rec.cnt + ' places (from cache)');
return true;
});
});
}).then(function (hit) {
if (hit) { cityFetching = false; return; }
@@ -2749,6 +2786,7 @@ function fetchSettlements() {
return loadCell(entry);
}).then(function () {
cityFetching = false;
manifestPersist(true);
publish();
openTileDB().then(function (db) {
if (!db) return;
@@ -3690,6 +3728,7 @@ function persistCarChains(fp) {
mCnts: Array.from(carGeo.motorway.cnts), mData: f32ToB64(carGeo.motorway.pts),
bCnts: Array.from(carGeo.secondary.cnts), bData: f32ToB64(carGeo.secondary.pts),
savedAt: Date.now() });
manifestPersist(true);
});
}