zink/animated_wallpaper
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

geo: cell manifest geo/cells_v1 - cached road/place fast paths check one small record instead of reading every multi-megabyte raw cell

Browse Source
This commit is contained in:
Richard Zink committed 2026-10-11 21:34:52 +02:00
1 parent 2d78c7081c
commit efd4db47cb
1 file changed
+83 -44
Download Patch File Download Diff File Expand all files Collapse all files
vendor/clouds3d.js
Vendored
+83 -44
View File
@@ -2206,11 +2206,42 @@ function parseOverpass(t) {
return (d && Array.isArray(d.elements)) ? d : null; return (d && Array.isArray(d.elements)) ? d : null;
} catch (e) { return null; } } catch (e) { return null; }
} }
// Fingerprint checks only need "is this record usable at all" - a full // Cell manifest: one tiny record that maps every cached Overpass cell to the
// JSON.parse of every cached cell just to hash its metadata is what made // time it was stored. The cached fast paths used to read every raw cell
// even a warm cache cycle expensive. // (hundreds of multi-megabyte JSON blobs) just to prove the chain cache is
function cheapOverpass(t) { // still current - the manifest answers the same question in one small read.
return !!t && t.length > 64 && t.indexOf('"elements"') >= 0; // Records are only ever written after parseOverpass() accepted them, so a
// poisoned cell cannot enter the manifest.
var cellManifest = null;
var _manifestPutAt = 0;
function loadManifest(db) {
return tileDbGet(db, 'geo/cells_v1').then(function (rec) {
if (!cellManifest) cellManifest = {};
// merge, never replace: notes taken this session may not be on disk
// yet (writes are throttled) and must not be lost.
if (rec && rec.map) {
for (var k in rec.map) {
if (!cellManifest[k] || cellManifest[k] < rec.map[k]) cellManifest[k] = rec.map[k];
}
}
return cellManifest;
});
}
function manifestNote(id, savedAt) {
if (!cellManifest) cellManifest = {};
if (cellManifest[id] !== savedAt) {
cellManifest[id] = savedAt;
manifestPersist(false);
}
}
function manifestPersist(force) {
if (!cellManifest) return;
var now = Date.now();
if (!force && now - _manifestPutAt < 3000) return;
_manifestPutAt = now;
openTileDB().then(function (db) {
if (db) tileDbPut(db, { id: 'geo/cells_v1', map: cellManifest, savedAt: now });
});
} }
function _opPump() { function _opPump() {
var now = Date.now(); var now = Date.now();
@@ -2503,6 +2534,7 @@ function fetchMotorways() {
if (cached) { if (cached) {
entry._cached = true; entry._cached = true;
fpParts.push(entry.id + ':' + rec.savedAt); fpParts.push(entry.id + ':' + rec.savedAt);
manifestNote(entry.id, rec.savedAt);
progCell(true); progCell(true);
return cached; return cached;
} }
@@ -2513,6 +2545,7 @@ function fetchMotorways() {
if (db) { if (db) {
var ts = Date.now(); var ts = Date.now();
fpParts.push(entry.id + ':' + ts); fpParts.push(entry.id + ':' + ts);
manifestNote(entry.id, ts);
tileDbPut(db, { id: entry.id, json: txt, savedAt: ts }); tileDbPut(db, { id: entry.id, json: txt, savedAt: ts });
progCell(false); progCell(false);
} }
@@ -2537,6 +2570,7 @@ function fetchMotorways() {
// worker for geometry. No stitching and no raw-JSON parsing at all. // worker for geometry. No stitching and no raw-JSON parsing at all.
openTileDB().then(function (db) { openTileDB().then(function (db) {
if (!db) return false; if (!db) return false;
return loadManifest(db).then(function (map) {
return tileDbGet(db, 'geo/roadchains_v1').then(function (rec) { return tileDbGet(db, 'geo/roadchains_v1').then(function (rec) {
if (!rec || !rec.mCnts || !rec.ids) return false; if (!rec || !rec.mCnts || !rec.ids) return false;
// Valid as long as the chains were built from the CURRENT query // Valid as long as the chains were built from the CURRENT query
@@ -2546,31 +2580,32 @@ function fetchMotorways() {
return rec.ids.indexOf(e.id) >= 0; return rec.ids.indexOf(e.id) >= 0;
}); });
if (!sameSet) return false; if (!sameSet) return false;
// The manifest lists the cached cells and their timestamps, so
// currency is checked without touching a single raw cell.
var newest = 0, missing = 0, total = CAR_QUERIES.length; var newest = 0, missing = 0, total = CAR_QUERIES.length;
return runPool(CAR_QUERIES, 8, function (e) { for (var i = 0; i < CAR_QUERIES.length; i++) {
return tileDbGet(db, e.id).then(function (r) { var at = map[CAR_QUERIES[i].id];
if (r && cheapOverpass(r.json)) { if (r.savedAt > newest) newest = r.savedAt; } if (!at) missing++;
else missing++; else if (at > newest) newest = at;
}); }
}).then(function () { // Reuse only when the raw set is essentially complete. A
// Reuse only when the raw set is essentially complete. A // partial build must stay rebuildable, otherwise a network
// partial build must stay rebuildable, otherwise a network // that was assembled while the mirrors were banning us would
// that was assembled while the mirrors were banning us would // be replayed forever and never grow.
// be replayed forever and never grow. if (newest > rec.savedAt) return false;
if (newest > rec.savedAt) return false; if (missing > Math.max(1, total * 0.1)) {
if (missing > Math.max(1, total * 0.1)) { dbgLog('[clouds3d] car layer: ' + missing + '/' + total +
dbgLog('[clouds3d] car layer: ' + missing + '/' + total + ' cells missing - rebuilding instead of replaying the cache');
' cells missing - rebuilding instead of replaying the cache'); return false;
return false; }
} carGeo = { motorway: { cnts: Int32Array.from(rec.mCnts), pts: b64ToF32(rec.mData) },
carGeo = { motorway: { cnts: Int32Array.from(rec.mCnts), pts: b64ToF32(rec.mData) }, secondary: { cnts: Int32Array.from(rec.bCnts), pts: b64ToF32(rec.bData) } };
secondary: { cnts: Int32Array.from(rec.bCnts), pts: b64ToF32(rec.bData) } }; kickCarBuild(false);
kickCarBuild(false); dbgLog('[clouds3d] car layer: ' + carGeo.motorway.cnts.length +
dbgLog('[clouds3d] car layer: ' + carGeo.motorway.cnts.length + ' motorway + ' + carGeo.secondary.cnts.length + ' chains (from cache' +
' motorway + ' + carGeo.secondary.cnts.length + ' chains (from cache' + (missing ? ', ' + missing + ' cells unavailable' : '') + ')');
(missing ? ', ' + missing + ' cells unavailable' : '') + ')'); return true;
return true; });
});
}); });
}).then(function (hit) { }).then(function (hit) {
if (hit) { carFetching = false; return; } if (hit) { carFetching = false; return; }
@@ -2579,6 +2614,7 @@ function fetchMotorways() {
return loadCell(entry); return loadCell(entry);
}).then(function () { }).then(function () {
carFetching = false; carFetching = false;
manifestPersist(true);
carPersistFp = geoFp(fpParts); carPersistFp = geoFp(fpParts);
carCycleFp = carPersistFp; carCycleFp = carPersistFp;
kickCarBuild(true); kickCarBuild(true);
@@ -2691,6 +2727,7 @@ function fetchSettlements() {
if (cached) { if (cached) {
entry._cached = true; entry._cached = true;
fpParts.push(entry.id + ':' + rec.savedAt); fpParts.push(entry.id + ':' + rec.savedAt);
manifestNote(entry.id, rec.savedAt);
progCell(true); progCell(true);
return cached; return cached;
} }
@@ -2701,6 +2738,7 @@ function fetchSettlements() {
if (db) { if (db) {
var ts = Date.now(); var ts = Date.now();
fpParts.push(entry.id + ':' + ts); fpParts.push(entry.id + ':' + ts);
manifestNote(entry.id, ts);
tileDbPut(db, { id: entry.id, json: txt, savedAt: ts }); tileDbPut(db, { id: entry.id, json: txt, savedAt: ts });
progCell(false); progCell(false);
} }
@@ -2725,23 +2763,22 @@ function fetchSettlements() {
// fast path: decoded settlement list cached with a matching cell fp // fast path: decoded settlement list cached with a matching cell fp
openTileDB().then(function (db) { openTileDB().then(function (db) {
if (!db) return false; if (!db) return false;
return loadManifest(db).then(function (map) {
return tileDbGet(db, 'geo/placecache_v1').then(function (rec) { return tileDbGet(db, 'geo/placecache_v1').then(function (rec) {
if (!rec || !rec.cnt || !rec.fp) return false; if (!rec || !rec.cnt || !rec.fp) return false;
var ok = true, parts = []; var parts = [];
return runPool(cells, 8, function (e) { for (var i = 0; i < cells.length; i++) {
if (!ok) return Promise.resolve(); var at = map[cells[i].id];
return tileDbGet(db, e.id).then(function (r) { if (!at) return false;
if (!r || !cheapOverpass(r.json)) ok = false; parts.push(cells[i].id + ':' + at);
else parts.push(e.id + ':' + r.savedAt); }
}); if (geoFp(parts) !== rec.fp) return false;
}).then(function () { cityGeo = b64ToF32(rec.data);
if (!ok || geoFp(parts) !== rec.fp) return false; kickCityBuild();
cityGeo = b64ToF32(rec.data); if (carGeo) kickCarBuild(false);
kickCityBuild(); dbgLog('[clouds3d] settlements: ' + rec.cnt + ' places (from cache)');
if (carGeo) kickCarBuild(false); return true;
dbgLog('[clouds3d] settlements: ' + rec.cnt + ' places (from cache)'); });
return true;
});
}); });
}).then(function (hit) { }).then(function (hit) {
if (hit) { cityFetching = false; return; } if (hit) { cityFetching = false; return; }
@@ -2749,6 +2786,7 @@ function fetchSettlements() {
return loadCell(entry); return loadCell(entry);
}).then(function () { }).then(function () {
cityFetching = false; cityFetching = false;
manifestPersist(true);
publish(); publish();
openTileDB().then(function (db) { openTileDB().then(function (db) {
if (!db) return; if (!db) return;
@@ -3690,6 +3728,7 @@ function persistCarChains(fp) {
mCnts: Array.from(carGeo.motorway.cnts), mData: f32ToB64(carGeo.motorway.pts), mCnts: Array.from(carGeo.motorway.cnts), mData: f32ToB64(carGeo.motorway.pts),
bCnts: Array.from(carGeo.secondary.cnts), bData: f32ToB64(carGeo.secondary.pts), bCnts: Array.from(carGeo.secondary.cnts), bData: f32ToB64(carGeo.secondary.pts),
savedAt: Date.now() }); savedAt: Date.now() });
manifestPersist(true);
}); });
} }