[core] Calculate size of an ambient cache without offline region's resources

CHANGELOG.md

@@ -36,6 +36,10 @@
 
   Before, it checked only text bounding boxes and thus label icons might have got cut off.
 
+- [core] Calculate size of an ambient cache without offline region's resources ([#15622](https://github.com/mapbox/mapbox-gl-native/pull/15622))
+
+  Resources that belong to an offline region, should not contribute to the amount of space available in the ambient cache.
+
 ### 🧩  Architectural changes
 
 - Changes to `MapSnapshotter` threading model ([#16268](https://github.com/mapbox/mapbox-gl-native/pull/16268))
@@ -46,6 +50,7 @@
 
 - Signature of a `MapSnapshotter`'s constructor has been changed
 - Signature for a `MapSnapshotter::snapshot` method has been changed
+- Size of an offline regions do not affect ambient cache size ([#15622](https://github.com/mapbox/mapbox-gl-native/pull/15622))
 
 ## maps-v1.3.0 (2020.02-relvanillashake)
 

metrics/binary-size/android-x86_64/metrics.json

@@ -3,7 +3,7 @@
         [
             "android-x86_64",
             "/tmp/attach/install/android-x86_64-release/lib/libmapbox-gl.so",
-            1988396
+            2008352
         ]
     ]
 }

platform/default/include/mbgl/storage/offline_database.hpp

@@ -3,7 +3,6 @@
 #include <mbgl/storage/resource.hpp>
 #include <mbgl/storage/offline.hpp>
 #include <mbgl/util/exception.hpp>
-#include <mbgl/util/noncopyable.hpp>
 #include <mbgl/util/optional.hpp>
 #include <mbgl/util/constants.hpp>
 #include <mbgl/util/mapbox.hpp>
@@ -36,7 +35,7 @@ struct MapboxTileLimitExceededException :  util::Exception {
     MapboxTileLimitExceededException() : util::Exception("Mapbox tile limit exceeded") {}
 };
 
-class OfflineDatabase : private util::noncopyable {
+class OfflineDatabase {
 public:
     OfflineDatabase(std::string path);
     ~OfflineDatabase();
@@ -97,6 +96,8 @@ class OfflineDatabase : private util::noncopyable {
     void reopenDatabaseReadOnly(bool readOnly);
 
 private:
+    class DatabaseSizeChangeStats;
+
     void initialize();
     void handleError(const mapbox::sqlite::Exception&, const char* action);
     void handleError(const util::IOException&, const char* action);
@@ -150,7 +151,37 @@ class OfflineDatabase : private util::noncopyable {
 
     optional<uint64_t> offlineMapboxTileCount;
 
-    bool evict(uint64_t neededFreeSize);
+    bool evict(uint64_t neededFreeSize, DatabaseSizeChangeStats& stats);
+
+    class DatabaseSizeChangeStats {
+    public:
+        explicit DatabaseSizeChangeStats(OfflineDatabase*);
+
+        // Returns difference between current database size and
+        // database size at the time of creation of this object.
+        int64_t diff() const;
+
+        // Returns how many bytes were released comparing to a database
+        // size at the time of creation of this object.
+        uint64_t bytesReleased() const;
+
+        // Returns page size for the database.
+        uint64_t pageSize() const;
+
+    private:
+        uint64_t pageSize_ = 0u;
+        uint64_t pageCount_ = 0u;
+        uint64_t initialSize_ = 0u;
+        OfflineDatabase* db = nullptr;
+    };
+
+    friend class DatabaseSizeChangeStats;
+
+    // Lazily initializes currentAmbientCacheSize.
+    std::exception_ptr initAmbientCacheSize();
+    optional<uint64_t> currentAmbientCacheSize;
+    void updateAmbientCacheSize(DatabaseSizeChangeStats&);
+
     bool autopack = true;
     bool readOnly = false;
 };

platform/default/src/mbgl/storage/offline_database.cpp

@@ -10,7 +10,6 @@
 #include <mbgl/storage/offline_schema.hpp>
 #include <mbgl/storage/merge_sideloaded.hpp>
 
-
 namespace mbgl {
 
 OfflineDatabase::OfflineDatabase(std::string path_)
@@ -311,9 +310,13 @@ std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource,
         size = compressed ? compressedData.size() : response.data->size();
     }
 
-    if (evict_ && !evict(size)) {
-        Log::Info(Event::Database, "Unable to make space for entry");
-        return { false, 0 };
+    optional<DatabaseSizeChangeStats> stats;
+    if (evict_) {
+        stats = DatabaseSizeChangeStats(this);
+        if (!evict(size, *stats)) {
+            Log::Info(Event::Database, "Unable to make space for entry");
+            return {false, 0};
+        }
     }
 
     bool inserted;
@@ -329,6 +332,10 @@ std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource,
                 compressed);
     }
 
+    if (stats) {
+        updateAmbientCacheSize(*stats);
+    }
+
     return { inserted, size };
 }
 
@@ -943,9 +950,11 @@ std::exception_ptr OfflineDatabase::deleteRegion(OfflineRegion&& region) try {
         query.run();
     }
 
-    evict(0);
+    DatabaseSizeChangeStats stats(this);
+    evict(0, stats);
     assert(db);
     if (autopack) vacuum();
+    updateAmbientCacheSize(stats);
 
     // Ensure that the cached offlineTileCount value is recalculated.
     offlineMapboxTileCount = nullopt;
@@ -1207,19 +1216,13 @@ T OfflineDatabase::getPragma(const char* sql) {
 // and as it approaches to the hard limit (i.e. the actual file size) we
 // delete an arbitrary number of old cache entries. The free pages approach saves
 // us from calling VACUUM or keeping a running total, which can be costly.
-bool OfflineDatabase::evict(uint64_t neededFreeSize) {
+bool OfflineDatabase::evict(uint64_t neededFreeSize, DatabaseSizeChangeStats& stats) {
     checkFlags();
+    uint64_t ambientCacheSize =
+        (initAmbientCacheSize() == nullptr) ? *currentAmbientCacheSize : maximumAmbientCacheSize;
+    uint64_t newAmbientCacheSize = ambientCacheSize + neededFreeSize + stats.pageSize();
 
-    uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size");
-    uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count");
-
-    auto usedSize = [&] {
-        return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count"));
-    };
-
-    // The addition of pageSize is a fudge factor to account for non `data` column
-    // size, and because pages can get fragmented on the database.
-    while (usedSize() + neededFreeSize + pageSize > maximumAmbientCacheSize) {
+    while (newAmbientCacheSize > maximumAmbientCacheSize) {
         // clang-format off
         mapbox::sqlite::Query accessedQuery{ getStatement(
             "SELECT max(accessed) "
@@ -1275,9 +1278,11 @@ bool OfflineDatabase::evict(uint64_t neededFreeSize) {
         tileQuery.run();
         const uint64_t tileChanges = tileQuery.changes();
 
+        // Update current ambient cache size, based on how many bytes were released.
+        newAmbientCacheSize = std::max<int64_t>(newAmbientCacheSize - stats.bytesReleased(), 0u);
+
         // The cached value of offlineTileCount does not need to be updated
         // here because only non-offline tiles can be removed by eviction.
-
         if (resourceChanges == 0 && tileChanges == 0) {
             return false;
         }
@@ -1286,18 +1291,75 @@ bool OfflineDatabase::evict(uint64_t neededFreeSize) {
     return true;
 }
 
+std::exception_ptr OfflineDatabase::initAmbientCacheSize() {
+    if (!currentAmbientCacheSize) {
+        try {
+            // clang-format off
+            mapbox::sqlite::Query query{ getStatement(
+            "SELECT SUM(data) "
+            "FROM ( "
+            "    SELECT SUM(IFNULL(LENGTH(data), 0) "
+            "               + IFNULL(LENGTH(id), 0) "
+            "               + IFNULL(LENGTH(url_template), 0) "
+            "               + IFNULL(LENGTH(pixel_ratio), 0) "
+            "               + IFNULL(LENGTH(x), 0) "
+            "               + IFNULL(LENGTH(y), 0) "
+            "               + IFNULL(LENGTH(z), 0) "
+            "               + IFNULL(LENGTH(expires), 0) "
+            "               + IFNULL(LENGTH(modified), 0) "
+            "               + IFNULL(LENGTH(etag), 0) "
+            "               + IFNULL(LENGTH(compressed), 0) "
+            "               + IFNULL(LENGTH(accessed), 0) "
+            "               + IFNULL(LENGTH(must_revalidate), 0) "
+            "               ) as data "
+            "    FROM tiles "
+            "    LEFT JOIN region_tiles "
+            "    ON tile_id = tiles.id "
+            "    WHERE tile_id IS NULL "
+            "  UNION ALL "
+            "    SELECT SUM(IFNULL(LENGTH(data), 0) "
+            "               + IFNULL(LENGTH(id), 0) "
+            "               + IFNULL(LENGTH(url), 0) "
+            "               + IFNULL(LENGTH(kind), 0) "
+            "               + IFNULL(LENGTH(expires), 0) "
+            "               + IFNULL(LENGTH(modified), 0) "
+            "               + IFNULL(LENGTH(etag), 0) "
+            "               + IFNULL(LENGTH(compressed), 0) "
+            "               + IFNULL(LENGTH(accessed), 0) "
+            "               + IFNULL(LENGTH(must_revalidate), 0) "
+            "               ) as data "
+            "    FROM resources "
+            "    LEFT JOIN region_resources "
+            "    ON resource_id = resources.id "
+            "    WHERE resource_id IS NULL "
+            ") ") };
+            // clang-format on
+            query.run();
+            currentAmbientCacheSize = query.get<int64_t>(0);
+        } catch (const mapbox::sqlite::Exception& ex) {
+            handleError(ex, "cannot get current ambient cache size");
+            return std::current_exception();
+        }
+    }
+
+    return nullptr;
+}
+
 std::exception_ptr OfflineDatabase::setMaximumAmbientCacheSize(uint64_t size) {
     uint64_t previousMaximumAmbientCacheSize = maximumAmbientCacheSize;
 
+    if (auto exception = initAmbientCacheSize()) {
+        return exception;
+    }
+
     try {
         maximumAmbientCacheSize = size;
 
-        uint64_t databaseSize = getPragma<int64_t>("PRAGMA page_size")
-            * getPragma<int64_t>("PRAGMA page_count");
-
-        if (databaseSize > maximumAmbientCacheSize) {
-            evict(0);
+        if (*currentAmbientCacheSize > maximumAmbientCacheSize) {
+            DatabaseSizeChangeStats stats(this);
+            evict(0, stats);
             if (autopack) vacuum();
+            updateAmbientCacheSize(stats);
         }
 
         return nullptr;
@@ -1395,4 +1457,33 @@ void OfflineDatabase::reopenDatabaseReadOnly(bool readOnly_) {
     }
 }
 
+OfflineDatabase::DatabaseSizeChangeStats::DatabaseSizeChangeStats(OfflineDatabase* db_) : db(db_) {
+    assert(db);
+    pageSize_ = db->getPragma<int64_t>("PRAGMA page_size");
+    pageCount_ = db->getPragma<int64_t>("PRAGMA page_count");
+    initialSize_ = pageSize_ * (pageCount_ - db->getPragma<int64_t>("PRAGMA freelist_count"));
+}
+
+uint64_t OfflineDatabase::DatabaseSizeChangeStats::pageSize() const {
+    return pageSize_;
+}
+
+int64_t OfflineDatabase::DatabaseSizeChangeStats::diff() const {
+    const uint64_t currentSize =
+        pageSize_ * (db->getPragma<int64_t>("PRAGMA page_count") - db->getPragma<int64_t>("PRAGMA freelist_count"));
+    return currentSize - initialSize_;
+}
+
+uint64_t OfflineDatabase::DatabaseSizeChangeStats::bytesReleased() const {
+    uint64_t currentSize = pageSize_ * (pageCount_ - db->getPragma<int64_t>("PRAGMA freelist_count"));
+    return std::max<int64_t>(initialSize_ - currentSize, 0u);
+}
+
+void OfflineDatabase::updateAmbientCacheSize(DatabaseSizeChangeStats& stats) {
+    assert(currentAmbientCacheSize);
+    if (currentAmbientCacheSize) {
+        *currentAmbientCacheSize = std::max<int64_t>(*currentAmbientCacheSize + stats.diff(), 0u);
+    }
+}
+
 } // namespace mbgl

test/storage/offline_database.test.cpp

@@ -714,19 +714,22 @@ TEST(OfflineDatabase, TEST_REQUIRES_WRITE(DeleteRegion)) {
         EXPECT_LT(sizeWithOneRegion, sizeWithTwoRegions);
         db.runPackDatabaseAutomatically(true);
         db.deleteRegion(std::move(*region2));
-        // The size of the database has shrunk right away.
+
+        // After clearing the cache, the size of the database
+        // should get back to the original size.
+        db.clearAmbientCache();
+
+        // The size of the database has shrunk right away after deleted region
+        // is evicted from an ambient cache.
         const size_t sizeWithoutRegions = util::read_file(filename).size();
-        ASSERT_EQ(0u, db.listRegions().value().size());
-        EXPECT_LT(sizeWithoutRegions, sizeWithOneRegion);
 
         // The tiles from the offline region will migrate to the
         // ambient cache and shrink the database to the maximum
         // size defined by default.
         EXPECT_LE(sizeWithoutRegions, util::DEFAULT_MAX_CACHE_SIZE);
 
-        // After clearing the cache, the size of the database
-        // should get back to the original size.
-        db.clearAmbientCache();
+        ASSERT_EQ(0u, db.listRegions().value().size());
+        EXPECT_LT(sizeWithoutRegions, sizeWithOneRegion);
     }
 
     EXPECT_EQ(initialSize, util::read_file(filename).size());
@@ -1059,7 +1062,8 @@ TEST(OfflineDatabase, PutEvictsLeastRecentlyUsedResources) {
     Response response;
     response.data = randomString(1024);
 
-    for (uint32_t i = 1; i <= 100; i++) {
+    // Add 101 resource to ambient cache, 1 over defined limit.
+    for (uint32_t i = 1; i <= 101; ++i) {
         Resource resource = Resource::style("http://example.com/"s + util::toString(i));
         db.put(resource, response);
         EXPECT_TRUE(bool(db.get(resource))) << i;
@@ -1070,6 +1074,45 @@ TEST(OfflineDatabase, PutEvictsLeastRecentlyUsedResources) {
     EXPECT_EQ(0u, log.uncheckedCount());
 }
 
+TEST(OfflineDatabase, OfflineRegionDoesNotAffectAmbientCacheSize) {
+    FixtureLog log;
+    OfflineDatabase db(":memory:");
+    unsigned dataSize = 1024u * 100u;
+    unsigned numberOfCachedResources = 2u;
+    unsigned databasePage = 4096u;
+    unsigned pageOverhead = numberOfCachedResources * databasePage;
+
+    // 200KB ambient cache limit + database page overhead.
+    db.setMaximumAmbientCacheSize(dataSize * numberOfCachedResources + pageOverhead);
+
+    Response response;
+    response.data = randomString(dataSize);
+
+    // First 100KB ambient cache resource.
+    db.put(Resource::style("http://example.com/ambient1.json"s), response);
+
+    OfflineTilePyramidRegionDefinition definition(
+        "http://example.com/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0, false);
+    auto region = db.createRegion(definition, OfflineRegionMetadata());
+
+    // 1MB of offline region data.
+    for (std::size_t i = 0; i < 5; ++i) {
+        const Resource tile = Resource::tile("mapbox://tile_" + std::to_string(i), 1, 0, 0, 0, Tileset::Scheme::XYZ);
+        db.putRegionResource(region->getID(), tile, response);
+
+        const Resource style = Resource::style("mapbox://style_" + std::to_string(i));
+        db.putRegionResource(region->getID(), style, response);
+    }
+
+    // Second 100KB ambient cache resource.
+    db.put(Resource::style("http://example.com/ambient2.json"s), response);
+
+    // Offline region resources should not affect ambient cache size.
+    EXPECT_TRUE(bool(db.get(Resource::style("http://example.com/ambient1.json"s))));
+    EXPECT_TRUE(bool(db.get(Resource::style("http://example.com/ambient2.json"s))));
+    EXPECT_EQ(0u, log.uncheckedCount());
+}
+
 TEST(OfflineDatabase, PutRegionResourceDoesNotEvict) {
     FixtureLog log;
     OfflineDatabase db(":memory:");
@@ -1098,7 +1141,8 @@ TEST(OfflineDatabase, PutFailsWhenEvictionInsuffices) {
     db.setMaximumAmbientCacheSize(1024 * 100);
 
     Response big;
-    big.data = randomString(1024 * 100);
+    // One page over the cache size.
+    big.data = randomString(1024 * 100 + 4096);
 
     EXPECT_FALSE(db.put(Resource::style("http://example.com/big"), big).first);