geo: cell manifest geo/cells_v1 - cached road/place fast paths check one small record instead of reading every multi-megabyte raw cell

vendor/clouds3d.js

@@ -2206,11 +2206,42 @@ function parseOverpass(t) {
         return (d && Array.isArray(d.elements)) ? d : null;
     } catch (e) { return null; }
 }
-// Fingerprint checks only need "is this record usable at all" - a full
+// Cell manifest: one tiny record that maps every cached Overpass cell to the
-// JSON.parse of every cached cell just to hash its metadata is what made
+// time it was stored. The cached fast paths used to read every raw cell
-// even a warm cache cycle expensive.
+// (hundreds of multi-megabyte JSON blobs) just to prove the chain cache is
-function cheapOverpass(t) {
+// still current - the manifest answers the same question in one small read.
-    return !!t && t.length > 64 && t.indexOf('"elements"') >= 0;
+// Records are only ever written after parseOverpass() accepted them, so a
+// poisoned cell cannot enter the manifest.
+var cellManifest = null;
+var _manifestPutAt = 0;
+function loadManifest(db) {
+    return tileDbGet(db, 'geo/cells_v1').then(function (rec) {
+        if (!cellManifest) cellManifest = {};
+        // merge, never replace: notes taken this session may not be on disk
+        // yet (writes are throttled) and must not be lost.
+        if (rec && rec.map) {
+            for (var k in rec.map) {
+                if (!cellManifest[k] || cellManifest[k] < rec.map[k]) cellManifest[k] = rec.map[k];
+            }
+        }
+        return cellManifest;
+    });
+}
+function manifestNote(id, savedAt) {
+    if (!cellManifest) cellManifest = {};
+    if (cellManifest[id] !== savedAt) {
+        cellManifest[id] = savedAt;
+        manifestPersist(false);
+    }
+}
+function manifestPersist(force) {
+    if (!cellManifest) return;
+    var now = Date.now();
+    if (!force && now - _manifestPutAt < 3000) return;
+    _manifestPutAt = now;
+    openTileDB().then(function (db) {
+        if (db) tileDbPut(db, { id: 'geo/cells_v1', map: cellManifest, savedAt: now });
+    });
 }
 function _opPump() {
     var now = Date.now();
@@ -2503,6 +2534,7 @@ function fetchMotorways() {
                     if (cached) {
                         entry._cached = true;
                         fpParts.push(entry.id + ':' + rec.savedAt);
+                        manifestNote(entry.id, rec.savedAt);
                         progCell(true);
                         return cached;
                     }
@@ -2513,6 +2545,7 @@ function fetchMotorways() {
                     if (db) {
                         var ts = Date.now();
                         fpParts.push(entry.id + ':' + ts);
+                        manifestNote(entry.id, ts);
                         tileDbPut(db, { id: entry.id, json: txt, savedAt: ts });
                         progCell(false);
                     }
@@ -2537,6 +2570,7 @@ function fetchMotorways() {
     // worker for geometry. No stitching and no raw-JSON parsing at all.
     openTileDB().then(function (db) {
         if (!db) return false;
+        return loadManifest(db).then(function (map) {
         return tileDbGet(db, 'geo/roadchains_v1').then(function (rec) {
             if (!rec || !rec.mCnts || !rec.ids) return false;
             // Valid as long as the chains were built from the CURRENT query
@@ -2546,13 +2580,14 @@ function fetchMotorways() {
                 return rec.ids.indexOf(e.id) >= 0;
             });
             if (!sameSet) return false;
+            // The manifest lists the cached cells and their timestamps, so
+            // currency is checked without touching a single raw cell.
             var newest = 0, missing = 0, total = CAR_QUERIES.length;
-            return runPool(CAR_QUERIES, 8, function (e) {
+            for (var i = 0; i < CAR_QUERIES.length; i++) {
-                return tileDbGet(db, e.id).then(function (r) {
+                var at = map[CAR_QUERIES[i].id];
-                    if (r && cheapOverpass(r.json)) { if (r.savedAt > newest) newest = r.savedAt; }
+                if (!at) missing++;
-                    else missing++;
+                else if (at > newest) newest = at;
-                });
+            }
-            }).then(function () {
             // Reuse only when the raw set is essentially complete. A
             // partial build must stay rebuildable, otherwise a network
             // that was assembled while the mirrors were banning us would
@@ -2579,6 +2614,7 @@ function fetchMotorways() {
             return loadCell(entry);
         }).then(function () {
             carFetching = false;
+            manifestPersist(true);
             carPersistFp = geoFp(fpParts);
             carCycleFp = carPersistFp;
             kickCarBuild(true);
@@ -2691,6 +2727,7 @@ function fetchSettlements() {
                     if (cached) {
                         entry._cached = true;
                         fpParts.push(entry.id + ':' + rec.savedAt);
+                        manifestNote(entry.id, rec.savedAt);
                         progCell(true);
                         return cached;
                     }
@@ -2701,6 +2738,7 @@ function fetchSettlements() {
                     if (db) {
                         var ts = Date.now();
                         fpParts.push(entry.id + ':' + ts);
+                        manifestNote(entry.id, ts);
                         tileDbPut(db, { id: entry.id, json: txt, savedAt: ts });
                         progCell(false);
                     }
@@ -2725,17 +2763,16 @@ function fetchSettlements() {
     // fast path: decoded settlement list cached with a matching cell fp
     openTileDB().then(function (db) {
         if (!db) return false;
+        return loadManifest(db).then(function (map) {
         return tileDbGet(db, 'geo/placecache_v1').then(function (rec) {
             if (!rec || !rec.cnt || !rec.fp) return false;
-            var ok = true, parts = [];
+            var parts = [];
-            return runPool(cells, 8, function (e) {
+            for (var i = 0; i < cells.length; i++) {
-                if (!ok) return Promise.resolve();
+                var at = map[cells[i].id];
-                return tileDbGet(db, e.id).then(function (r) {
+                if (!at) return false;
-                    if (!r || !cheapOverpass(r.json)) ok = false;
+                parts.push(cells[i].id + ':' + at);
-                    else parts.push(e.id + ':' + r.savedAt);
+            }
-                });
+            if (geoFp(parts) !== rec.fp) return false;
-            }).then(function () {
-                if (!ok || geoFp(parts) !== rec.fp) return false;
             cityGeo = b64ToF32(rec.data);
             kickCityBuild();
             if (carGeo) kickCarBuild(false);
@@ -2749,6 +2786,7 @@ function fetchSettlements() {
             return loadCell(entry);
         }).then(function () {
             cityFetching = false;
+            manifestPersist(true);
             publish();
             openTileDB().then(function (db) {
                 if (!db) return;
@@ -3690,6 +3728,7 @@ function persistCarChains(fp) {
             mCnts: Array.from(carGeo.motorway.cnts), mData: f32ToB64(carGeo.motorway.pts),
             bCnts: Array.from(carGeo.secondary.cnts), bData: f32ToB64(carGeo.secondary.pts),
             savedAt: Date.now() });
+        manifestPersist(true);
     });
 }