Bug 1980177: Skip over origins and log a telemetry for origins with unknown metadata when performing idb maintenance.r=asuth,dom-storage-reviewers,toolkit-telemetry-reviewers,TravisLong

Created few telemetry metrics to track the number of times we need to load metadata from metadata file, restore metadata and
the number of times we encounter an unknown metadata. This patch also introduces new QuotaManager::LoadFullOriginMetadataWithRestoreAndStatus
which is similar to LoadFullOriginMetadataWithRestore but also returns whether the metadata file was restored or not.

Differential Revision: https://phabricator.services.mozilla.com/D260057

Author: hsingh-moz

dom/indexedDB/ActorsParent.cpp

@@ -148,6 +148,7 @@
 #include "mozilla/dom/quota/UniversalDirectoryLock.h"
 #include "mozilla/dom/quota/UsageInfo.h"
 #include "mozilla/fallible.h"
+#include "mozilla/glean/DomIndexedDBMetrics.h"
 #include "mozilla/ipc/BackgroundParent.h"
 #include "mozilla/ipc/BackgroundUtils.h"
 #include "mozilla/ipc/InputStreamParams.h"
@@ -13199,7 +13200,7 @@ RefPtr<UniversalDirectoryLockPromise> Maintenance::OpenStorageDirectory(
   QuotaManager* quotaManager = QuotaManager::Get();
   MOZ_ASSERT(quotaManager);
 
-  // Return a shared lock for <profile>/storage/*/*/idb
+  // Return a shared lock for <profile>/storage/repo(aPersistenceScope)/*/idb
   return quotaManager->OpenStorageDirectory(
       aPersistenceScope, OriginScope::FromNull(),
       ClientStorageScope::CreateFromClient(Client::IDB),
@@ -13363,6 +13364,11 @@ nsresult Maintenance::DirectoryWork() {
       continue;
     }
 
+    // For all non-persistent types, temporary storage must already be
+    // initialized
+    MOZ_DIAGNOSTIC_ASSERT(
+        persistent || quotaManager->IsTemporaryStorageInitializedInternal());
+
     // XXX persistenceType == PERSISTENCE_TYPE_PERSISTENT shouldn't be a special
     // case...
     const auto persistenceTypeString =
@@ -13418,6 +13424,8 @@ nsresult Maintenance::DirectoryWork() {
               if (!persistent &&
                   !quotaManager->IsTemporaryOriginInitializedInternal(
                       metadata)) {
+                glean::idb_maintenance::fallback_fullrestore_metadata.Add();
+
                 // XXX GetOriginMetadata, which skips loading the metadata file
                 // and instead relies on parsing the origin directory name and
                 // reconstructing the principal, may produce a different origin
@@ -13430,10 +13438,26 @@ nsresult Maintenance::DirectoryWork() {
                 // In the future, it would be useful to report anonymized
                 // origin strings via telemetry to help investigate and
                 // eventually fix this mismatch.
-                QM_TRY_UNWRAP(
-                    metadata,
-                    quotaManager->LoadFullOriginMetadataWithRestore(originDir),
+
+                QM_TRY_INSPECT(
+                    const auto& fullmetadataAndStatus,
+                    quotaManager->LoadFullOriginMetadataWithRestoreAndStatus(
+                        originDir),
                     Ok{});
+
+                metadata = fullmetadataAndStatus.first;
+                if (fullmetadataAndStatus.second) {
+                  /* metadata was restored */
+                  glean::idb_maintenance::metadata_restored.Add();
+                }
+
+                QM_TRY(OkIf(quotaManager->IsTemporaryOriginInitializedInternal(
+                           metadata)),
+                       /* unexpected but still fine to just skip it */ Ok{},
+                       /* increment telemetry in case of failure */
+                       [](const auto&) {
+                         glean::idb_maintenance::unknown_metadata.Add();
+                       });
               }
 
               // We now use a dedicated repository for private browsing
@@ -13508,7 +13532,6 @@ nsresult Maintenance::DirectoryWork() {
                 if (!persistent) {
                   auto maybeOriginStateMetadata =
                       quotaManager->GetOriginStateMetadata(metadata);
-
                   auto originStateMetadata = maybeOriginStateMetadata.extract();
 
                   // Skip origin maintenance if the origin hasn't been accessed

dom/indexedDB/metrics.yaml

@@ -0,0 +1,60 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Adding a new metric? We have docs for that!
+# https://firefox-source-docs.mozilla.org/toolkit/components/glean/user/new_definitions_file.html
+
+---
+$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
+$tags:
+  - 'Core :: Storage: IndexedDB'
+
+idb.maintenance:
+  fallback_fullrestore_metadata:
+    type: counter
+    description: >
+      Tracks the number of times we need to fallback to restore metadata object by reading
+      metadata-v2 file on disk when performing idle-maintenance on IDB database.
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1980177
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1980177
+    data_sensitivity:
+      - technical
+    notification_emails:
+      - storage-telemetry@mozilla.com
+    expires: never
+
+  metadata_restored:
+    type: counter
+    description: >
+      Counts the number of times we failed to load metadata object and had to restore it
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1980177
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1980177
+    data_sensitivity:
+      - technical
+    notification_emails:
+      - storage-telemetry@mozilla.com
+    expires: never
+
+  unknown_metadata:
+    type: counter
+    description: >
+      Increments upon encountering an unknown metadata object when performing
+      idle-maintenance on IDB database. During idle-maintenance, we iterate
+      over idb directories/files and construct a metadata object by calling
+      QuotaManager::GetOriginMetadata or LoadFullOriginMetadataWithRestore.
+      It seems that in some cases, the metadata object we get here can be
+      invalid and this metric is used to track those instances.
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1980177
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1980177
+    data_sensitivity:
+      - technical
+    notification_emails:
+      - storage-telemetry@mozilla.com
+    expires: never

dom/quota/ActorsParent.cpp

@@ -3755,6 +3755,16 @@ Result<FullOriginMetadata, nsresult> QuotaManager::LoadFullOriginMetadata(
 
 Result<FullOriginMetadata, nsresult>
 QuotaManager::LoadFullOriginMetadataWithRestore(nsIFile* aDirectory) {
+  QM_TRY_INSPECT(const auto& result,
+                 LoadFullOriginMetadataWithRestoreAndStatus(aDirectory));
+
+  // The first element of the pair is the FullOriginMetadata, the second is the
+  // restore status.
+  return result.first;
+}
+
+Result<std::pair<FullOriginMetadata, bool /* restore status */>, nsresult>
+QuotaManager::LoadFullOriginMetadataWithRestoreAndStatus(nsIFile* aDirectory) {
   // XXX Once the persistence type is stored in the metadata file, this block
   // for getting the persistence type from the parent directory name can be
   // removed.
@@ -3767,16 +3777,23 @@ QuotaManager::LoadFullOriginMetadataWithRestore(nsIFile* aDirectory) {
 
   const auto& persistenceType = maybePersistenceType.value();
 
-  QM_TRY_RETURN(QM_OR_ELSE_WARN(
-      // Expression.
-      LoadFullOriginMetadata(aDirectory, persistenceType),
-      // Fallback.
-      ([&aDirectory, &persistenceType,
-        this](const nsresult rv) -> Result<FullOriginMetadata, nsresult> {
-        QM_TRY(MOZ_TO_RESULT(RestoreDirectoryMetadata2(aDirectory)));
+  bool restored = false;
 
-        QM_TRY_RETURN(LoadFullOriginMetadata(aDirectory, persistenceType));
-      })));
+  QM_TRY_INSPECT(
+      const auto& fullOriginMetadata,
+      QM_OR_ELSE_WARN(
+          // Expression.
+          LoadFullOriginMetadata(aDirectory, persistenceType),
+          // Fallback.
+          ([&aDirectory, &persistenceType, &restored,
+            this](const nsresult rv) -> Result<FullOriginMetadata, nsresult> {
+            restored = true;
+            QM_TRY(MOZ_TO_RESULT(RestoreDirectoryMetadata2(aDirectory)));
+
+            QM_TRY_RETURN(LoadFullOriginMetadata(aDirectory, persistenceType));
+          })));
+
+  return std::make_pair(fullOriginMetadata, restored);
 }
 
 Result<OriginMetadata, nsresult> QuotaManager::GetOriginMetadata(

dom/quota/QuotaManager.h

@@ -304,6 +304,9 @@ class QuotaManager final : public BackgroundThreadObject {
   Result<FullOriginMetadata, nsresult> LoadFullOriginMetadataWithRestore(
       nsIFile* aDirectory);
 
+  Result<std::pair<FullOriginMetadata, bool /* restore status */>, nsresult>
+  LoadFullOriginMetadataWithRestoreAndStatus(nsIFile* aDirectory);
+
   Result<OriginMetadata, nsresult> GetOriginMetadata(nsIFile* aDirectory);
 
   Result<Ok, nsresult> RemoveOriginDirectory(nsIFile& aDirectory);

toolkit/components/glean/metrics_index.py

@@ -29,6 +29,7 @@
     "dom/canvas/metrics.yaml",
     "dom/crypto/metrics.yaml",
     "dom/geolocation/metrics.yaml",
+    "dom/indexedDB/metrics.yaml",
     "dom/localstorage/metrics.yaml",
     "dom/media/eme/metrics.yaml",
     "dom/media/hls/metrics.yaml",