ngclient: Support "lazy_refresh"

This configuration avoids fetching remote metadata if local toplevel
metadata is valid.
* This is _not_ TUF spec compliant
* May lead to issues on download_target if delegated metadata is expired
* Requires the repository to have a reasonably quick timestamp expiry
  (otherwise clients could be unaware of updates for a long time)
* Because of that should likely be accompanied with client application
  doing "non-lazy" refreshes occasionally, regardless of expiry
* Still, could be a useful feature for clients of repositories that have
  slow-moving metadata in situations where clients are content with a
  delay of timestamp expiry time for any updates.

The implementation is aiming to be as simple as possible: this should
only be done if the code remains understandable.

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

tests/test_updater_top_level_update.py

@@ -737,6 +737,111 @@ def test_load_metadata_from_cache(self, wrapped_open: MagicMock) -> None:
         expected_calls = [("root", 2), ("timestamp", None)]
         self.assertListEqual(self.sim.fetch_tracker.metadata, expected_calls)
 
+    @patch.object(datetime, "datetime", wraps=datetime.datetime)
+    def test_refresh_with_lazy_refresh(self, mock_time: Mock) -> None:
+        # make v1 metadata expire a little earlier
+        self.sim.timestamp.expires = self.sim.safe_expiry - datetime.timedelta(
+            days=7
+        )
+        self.sim.targets.expires = self.sim.safe_expiry - datetime.timedelta(
+            days=5
+        )
+
+        # Make a successful update of valid metadata which stores it in cache
+        updater = self._init_updater()
+        updater.config.lazy_refresh = True
+        updater.refresh()
+
+        # Clean up fetch tracker data
+        self.sim.fetch_tracker.metadata.clear()
+
+        ### Refresh succeeds and is lazy
+
+        # create timestamp v2 in repository
+        self.sim.timestamp.version += 1
+        self.sim.timestamp.expires = self.sim.safe_expiry
+
+        # Perform a second update. lazy_refresh means no metadata fetches are
+        # expected this time
+        updater = self._init_updater()
+        updater.config.lazy_refresh = True
+        updater.refresh()
+        self.assertListEqual(self.sim.fetch_tracker.metadata, [])
+
+        ### Refresh succeeds but is not lazy
+
+        # Mock time so local timestamp has expired but new timestamp has not.
+        # lazy refresh fails, so we end up with normal refresh
+        mock_time.utcnow.return_value = (
+            self.sim.safe_expiry - datetime.timedelta(days=6)
+        )
+        with patch("datetime.datetime", mock_time):
+            updater = self._init_updater()
+            updater.config.lazy_refresh = True
+            updater.refresh()
+
+        expected_calls = [("root", 2), ("timestamp", None)]
+        self.assertListEqual(self.sim.fetch_tracker.metadata, expected_calls)
+
+        # Clean up fetch tracker data
+        self.sim.fetch_tracker.metadata.clear()
+
+        ### Refresh fails and is not lazy
+
+        # Mock time so local targets has expired. lazy refresh fails, so we
+        # end up with normal refresh which also fails
+        mock_time.utcnow.return_value = (
+            self.sim.safe_expiry - datetime.timedelta(days=4)
+        )
+        with patch("datetime.datetime", mock_time):
+            updater = self._init_updater()
+            updater.config.lazy_refresh = True
+            with self.assertRaises(ExpiredMetadataError):
+                updater.refresh()
+
+        expected_calls = [
+            ("root", 2),
+            ("timestamp", None),
+            ("targets", 1),
+        ]
+        self.assertListEqual(self.sim.fetch_tracker.metadata, expected_calls)
+
+        # Clean up fetch tracker data
+        self.sim.fetch_tracker.metadata.clear()
+
+        ### Refresh succeeds but is not lazy
+
+        # create targets v2 in repository
+        self.sim.targets.version += 1
+        self.sim.targets.expires = self.sim.safe_expiry
+        self.sim.update_snapshot()
+
+        # Mock time so that lazy refresh fails but the normal refresh succeeds
+        with patch("datetime.datetime", mock_time):
+            updater = self._init_updater()
+            updater.config.lazy_refresh = True
+            updater.refresh()
+
+        expected_calls = [
+            ("root", 2),
+            ("timestamp", None),
+            ("snapshot", 2),
+            ("targets", 2),
+        ]
+        self.assertListEqual(self.sim.fetch_tracker.metadata, expected_calls)
+
+        # Clean up fetch tracker data
+        self.sim.fetch_tracker.metadata.clear()
+
+        ### Refresh succeeds and is lazy
+
+        with patch("datetime.datetime", mock_time):
+            updater = self._init_updater()
+            updater.config.lazy_refresh = True
+            updater.refresh()
+
+        self.assertListEqual(self.sim.fetch_tracker.metadata, [])
+
     @patch.object(datetime, "datetime", wraps=datetime.datetime)
     def test_expired_metadata(self, mock_time: Mock) -> None:
         """Verifies that expired local timestamp/snapshot can be used for

tuf/ngclient/config.py

@@ -23,7 +23,15 @@ class UpdaterConfig:
             are used, target download URLs are formed by prefixing the filename
             with a hash digest of file content by default. This can be
             overridden by setting ``prefix_targets_with_hash`` to ``False``.
-
+        lazy_refresh: Do not fetch metadata from remote if the local metadata
+            is still valid. Setting lazy_refresh to True means refresh() no
+            longer implements the full client workflow that is described in the
+            specification, and should only be used with repositories that
+            suggest using it:
+             * The client may stay unaware of metadata updates for the
+               expiry periods (typically timestamp expiry period).
+             * Repository maintenance has some additional requirements as the
+               clients may operate with older metadata.
     """
 
     max_root_rotations: int = 32
@@ -33,3 +41,4 @@ class UpdaterConfig:
     snapshot_max_length: int = 2000000  # bytes
     targets_max_length: int = 5000000  # bytes
     prefix_targets_with_hash: bool = True
+    lazy_refresh = False

tuf/ngclient/updater.py

@@ -53,7 +53,8 @@
     Targets,
     Timestamp,
 )
-from tuf.ngclient._internal import requests_fetcher, trusted_metadata_set
+from tuf.ngclient._internal import requests_fetcher
+from tuf.ngclient._internal.trusted_metadata_set import TrustedMetadataSet
 from tuf.ngclient.config import UpdaterConfig
 from tuf.ngclient.fetcher import FetcherInterface
 
@@ -101,7 +102,7 @@ def __init__(
 
         # Read trusted local root metadata
         data = self._load_local_metadata(Root.type)
-        self._trusted_set = trusted_metadata_set.TrustedMetadataSet(data)
+        self._trusted_set = TrustedMetadataSet(data)
         self._fetcher = fetcher or requests_fetcher.RequestsFetcher()
         self.config = config or UpdaterConfig()
 
@@ -129,6 +130,23 @@ def refresh(self) -> None:
             DownloadError: Download of a metadata file failed in some way
         """
 
+        if self.config.lazy_refresh:
+            try:
+                # Try loading only local data
+                data = self._load_local_metadata(Timestamp.type)
+                self._trusted_set.update_timestamp(data)
+                data = self._load_local_metadata(Snapshot.type)
+                self._trusted_set.update_snapshot(data, trusted=True)
+                data = self._load_local_metadata(Targets.type)
+                self._trusted_set.update_delegated_targets(
+                    data, Targets.type, Root.type
+                )
+                return
+            except (OSError, exceptions.RepositoryError):
+                # Failed: reset _trusted_set, continue with vanilla refresh
+                data = self._load_local_metadata(Root.type)
+                self._trusted_set = TrustedMetadataSet(data)
+
         self._load_root()
         self._load_timestamp()
         self._load_snapshot()