diff --git a/vendor/clouds3d.js b/vendor/clouds3d.js index e7af4ff..08b801f 100644 --- a/vendor/clouds3d.js +++ b/vendor/clouds3d.js @@ -2206,11 +2206,42 @@ function parseOverpass(t) { return (d && Array.isArray(d.elements)) ? d : null; } catch (e) { return null; } } -// Fingerprint checks only need "is this record usable at all" - a full -// JSON.parse of every cached cell just to hash its metadata is what made -// even a warm cache cycle expensive. -function cheapOverpass(t) { - return !!t && t.length > 64 && t.indexOf('"elements"') >= 0; +// Cell manifest: one tiny record that maps every cached Overpass cell to the +// time it was stored. The cached fast paths used to read every raw cell +// (hundreds of multi-megabyte JSON blobs) just to prove the chain cache is +// still current - the manifest answers the same question in one small read. +// Records are only ever written after parseOverpass() accepted them, so a +// poisoned cell cannot enter the manifest. +var cellManifest = null; +var _manifestPutAt = 0; +function loadManifest(db) { + return tileDbGet(db, 'geo/cells_v1').then(function (rec) { + if (!cellManifest) cellManifest = {}; + // merge, never replace: notes taken this session may not be on disk + // yet (writes are throttled) and must not be lost. + if (rec && rec.map) { + for (var k in rec.map) { + if (!cellManifest[k] || cellManifest[k] < rec.map[k]) cellManifest[k] = rec.map[k]; + } + } + return cellManifest; + }); +} +function manifestNote(id, savedAt) { + if (!cellManifest) cellManifest = {}; + if (cellManifest[id] !== savedAt) { + cellManifest[id] = savedAt; + manifestPersist(false); + } +} +function manifestPersist(force) { + if (!cellManifest) return; + var now = Date.now(); + if (!force && now - _manifestPutAt < 3000) return; + _manifestPutAt = now; + openTileDB().then(function (db) { + if (db) tileDbPut(db, { id: 'geo/cells_v1', map: cellManifest, savedAt: now }); + }); } function _opPump() { var now = Date.now(); @@ -2503,6 +2534,7 @@ function fetchMotorways() { if (cached) { entry._cached = true; fpParts.push(entry.id + ':' + rec.savedAt); + manifestNote(entry.id, rec.savedAt); progCell(true); return cached; } @@ -2513,6 +2545,7 @@ function fetchMotorways() { if (db) { var ts = Date.now(); fpParts.push(entry.id + ':' + ts); + manifestNote(entry.id, ts); tileDbPut(db, { id: entry.id, json: txt, savedAt: ts }); progCell(false); } @@ -2537,6 +2570,7 @@ function fetchMotorways() { // worker for geometry. No stitching and no raw-JSON parsing at all. openTileDB().then(function (db) { if (!db) return false; + return loadManifest(db).then(function (map) { return tileDbGet(db, 'geo/roadchains_v1').then(function (rec) { if (!rec || !rec.mCnts || !rec.ids) return false; // Valid as long as the chains were built from the CURRENT query @@ -2546,31 +2580,32 @@ function fetchMotorways() { return rec.ids.indexOf(e.id) >= 0; }); if (!sameSet) return false; + // The manifest lists the cached cells and their timestamps, so + // currency is checked without touching a single raw cell. var newest = 0, missing = 0, total = CAR_QUERIES.length; - return runPool(CAR_QUERIES, 8, function (e) { - return tileDbGet(db, e.id).then(function (r) { - if (r && cheapOverpass(r.json)) { if (r.savedAt > newest) newest = r.savedAt; } - else missing++; - }); - }).then(function () { - // Reuse only when the raw set is essentially complete. A - // partial build must stay rebuildable, otherwise a network - // that was assembled while the mirrors were banning us would - // be replayed forever and never grow. - if (newest > rec.savedAt) return false; - if (missing > Math.max(1, total * 0.1)) { - dbgLog('[clouds3d] car layer: ' + missing + '/' + total + - ' cells missing - rebuilding instead of replaying the cache'); - return false; - } - carGeo = { motorway: { cnts: Int32Array.from(rec.mCnts), pts: b64ToF32(rec.mData) }, - secondary: { cnts: Int32Array.from(rec.bCnts), pts: b64ToF32(rec.bData) } }; - kickCarBuild(false); - dbgLog('[clouds3d] car layer: ' + carGeo.motorway.cnts.length + - ' motorway + ' + carGeo.secondary.cnts.length + ' chains (from cache' + - (missing ? ', ' + missing + ' cells unavailable' : '') + ')'); - return true; - }); + for (var i = 0; i < CAR_QUERIES.length; i++) { + var at = map[CAR_QUERIES[i].id]; + if (!at) missing++; + else if (at > newest) newest = at; + } + // Reuse only when the raw set is essentially complete. A + // partial build must stay rebuildable, otherwise a network + // that was assembled while the mirrors were banning us would + // be replayed forever and never grow. + if (newest > rec.savedAt) return false; + if (missing > Math.max(1, total * 0.1)) { + dbgLog('[clouds3d] car layer: ' + missing + '/' + total + + ' cells missing - rebuilding instead of replaying the cache'); + return false; + } + carGeo = { motorway: { cnts: Int32Array.from(rec.mCnts), pts: b64ToF32(rec.mData) }, + secondary: { cnts: Int32Array.from(rec.bCnts), pts: b64ToF32(rec.bData) } }; + kickCarBuild(false); + dbgLog('[clouds3d] car layer: ' + carGeo.motorway.cnts.length + + ' motorway + ' + carGeo.secondary.cnts.length + ' chains (from cache' + + (missing ? ', ' + missing + ' cells unavailable' : '') + ')'); + return true; + }); }); }).then(function (hit) { if (hit) { carFetching = false; return; } @@ -2579,6 +2614,7 @@ function fetchMotorways() { return loadCell(entry); }).then(function () { carFetching = false; + manifestPersist(true); carPersistFp = geoFp(fpParts); carCycleFp = carPersistFp; kickCarBuild(true); @@ -2691,6 +2727,7 @@ function fetchSettlements() { if (cached) { entry._cached = true; fpParts.push(entry.id + ':' + rec.savedAt); + manifestNote(entry.id, rec.savedAt); progCell(true); return cached; } @@ -2701,6 +2738,7 @@ function fetchSettlements() { if (db) { var ts = Date.now(); fpParts.push(entry.id + ':' + ts); + manifestNote(entry.id, ts); tileDbPut(db, { id: entry.id, json: txt, savedAt: ts }); progCell(false); } @@ -2725,23 +2763,22 @@ function fetchSettlements() { // fast path: decoded settlement list cached with a matching cell fp openTileDB().then(function (db) { if (!db) return false; + return loadManifest(db).then(function (map) { return tileDbGet(db, 'geo/placecache_v1').then(function (rec) { if (!rec || !rec.cnt || !rec.fp) return false; - var ok = true, parts = []; - return runPool(cells, 8, function (e) { - if (!ok) return Promise.resolve(); - return tileDbGet(db, e.id).then(function (r) { - if (!r || !cheapOverpass(r.json)) ok = false; - else parts.push(e.id + ':' + r.savedAt); - }); - }).then(function () { - if (!ok || geoFp(parts) !== rec.fp) return false; - cityGeo = b64ToF32(rec.data); - kickCityBuild(); - if (carGeo) kickCarBuild(false); - dbgLog('[clouds3d] settlements: ' + rec.cnt + ' places (from cache)'); - return true; - }); + var parts = []; + for (var i = 0; i < cells.length; i++) { + var at = map[cells[i].id]; + if (!at) return false; + parts.push(cells[i].id + ':' + at); + } + if (geoFp(parts) !== rec.fp) return false; + cityGeo = b64ToF32(rec.data); + kickCityBuild(); + if (carGeo) kickCarBuild(false); + dbgLog('[clouds3d] settlements: ' + rec.cnt + ' places (from cache)'); + return true; + }); }); }).then(function (hit) { if (hit) { cityFetching = false; return; } @@ -2749,6 +2786,7 @@ function fetchSettlements() { return loadCell(entry); }).then(function () { cityFetching = false; + manifestPersist(true); publish(); openTileDB().then(function (db) { if (!db) return; @@ -3690,6 +3728,7 @@ function persistCarChains(fp) { mCnts: Array.from(carGeo.motorway.cnts), mData: f32ToB64(carGeo.motorway.pts), bCnts: Array.from(carGeo.secondary.cnts), bData: f32ToB64(carGeo.secondary.pts), savedAt: Date.now() }); + manifestPersist(true); }); }