From 1b49f08f6373feb0a09832d6208b30d9317ebbaf Mon Sep 17 00:00:00 2001 From: marinamoore Date: Thu, 6 Aug 2020 11:35:01 -0700 Subject: [PATCH 1/9] Add targets map file to single-repository updater This commit adds support for TAP 13 to the Updater class. TAP 13 allows a client to specify which targets metadata on the repository they would like to use as the top-level targets metadata. This mechanism allows repository managers to split a repository into multiple namespaces and for clients to specify which of these namespaces to trust. This commit implements this feature by reading the targets map file and using the indicated metadata as top-level targets metadata. Signed-off-by: marinamoore --- tuf/client/updater.py | 35 ++++++++++++++++++++++++++++++----- tuf/formats.py | 6 ++++++ tuf/keydb.py | 40 +++++++++++++++++++++++++++++++++++++++- tuf/roledb.py | 7 ++++++- 4 files changed, 81 insertions(+), 7 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 564e285efd..57da6ead55 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -626,7 +626,7 @@ class Updater(object): http://www.python.org/dev/peps/pep-0008/#method-names-and-instance-variables """ - def __init__(self, repository_name, repository_mirrors): + def __init__(self, repository_name, repository_mirrors, targets_map_file = None): """ Constructor. Instantiating an updater object causes all the metadata @@ -694,6 +694,23 @@ def __init__(self, repository_name, repository_mirrors): # Save the validated arguments. self.repository_name = repository_name self.mirrors = repository_mirrors + self.targets_map_file = None + + if targets_map_file is not None: + # Is 'targets_map_file' a path? If not, raise + # 'securesystemslib.exceptions.FormatError'. The actual content of the map + # file is validated later on in this method. + securesystemslib.formats.PATH_SCHEMA.check_match(targets_map_file) + + try: + self.targets_map_file = securesystemslib.util.load_json_file(targets_map_file) + + except (securesystemslib.exceptions.Error) as e: + raise tuf.exceptions.Error('Cannot load the targets map file: ' + str(e)) + + # Raise securesystemslib.exceptions.FormatError if the targets map file is + # improperly formatted. + tuf.formats.TARGETS_MAPFILE_SCHEMA.check_match(self.targets_map_file) # Store the trusted metadata read from disk. self.metadata = {} @@ -822,7 +839,12 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): # Save and construct the full metadata path. metadata_directory = self.metadata_directory[metadata_set] - metadata_filename = metadata_role + '.json' + # For top-level targets, the targets map file may overwrite the + # targets metadata on the repository + if metadata_role == 'targets' and self.targets_map_file is not None: + metadata_filename = self.targets_map_file['targets_filename'] + '.json' + else: + metadata_filename = metadata_role + '.json' metadata_filepath = os.path.join(metadata_directory, metadata_filename) # Ensure the metadata path is valid/exists, else ignore the call. @@ -900,10 +922,10 @@ def _rebuild_key_and_role_db(self): # repository is first instantiated. Due to this setup, reloading delegated # roles is not required here. tuf.keydb.create_keydb_from_root_metadata(self.metadata['current']['root'], - self.repository_name) + self.repository_name, self.targets_map_file) tuf.roledb.create_roledb_from_root_metadata(self.metadata['current']['root'], - self.repository_name) + self.repository_name, self.targets_map_file) @@ -1785,7 +1807,10 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None): # Construct the metadata filename as expected by the download/mirror # modules. - metadata_filename = metadata_role + '.json' + if self.targets_map_file is not None and metadata_role == 'targets': + metadata_filename = self.targets_map_file['targets_filename'] + '.json' + else: + metadata_filename = metadata_role + '.json' # Attempt a file download from each mirror until the file is downloaded and # verified. If the signature of the downloaded file is valid, proceed, diff --git a/tuf/formats.py b/tuf/formats.py index d289ddb0b4..082912bc22 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -276,6 +276,12 @@ key_schema = NAME_SCHEMA, value_schema = SCHEMA.ListOf(securesystemslib.formats.URL_SCHEMA)) +# An object containing the targets map file. The format of the targets +# map file is covered in TAP 13 +TARGETS_MAPFILE_SCHEMA = SCHEMA.Object( + targets_filename = NAME_SCHEMA, + keys = KEYDICT_SCHEMA) + # An object containing the map file's "mapping" attribute. MAPPING_SCHEMA = SCHEMA.ListOf(SCHEMA.Object( paths = RELPATHS_SCHEMA, diff --git a/tuf/keydb.py b/tuf/keydb.py index bc306bac25..790a81c4d6 100755 --- a/tuf/keydb.py +++ b/tuf/keydb.py @@ -60,7 +60,7 @@ _keydb_dict['default'] = {} -def create_keydb_from_root_metadata(root_metadata, repository_name='default'): +def create_keydb_from_root_metadata(root_metadata, repository_name='default', targets_map_file = None): """ Populate the key database with the unique keys found in 'root_metadata'. @@ -111,6 +111,44 @@ def create_keydb_from_root_metadata(root_metadata, repository_name='default'): else: create_keydb(repository_name) + + if targets_map_file is not None: + tuf.formats.TARGETS_MAPFILE_SCHEMA.check_match(targets_map_file) + + for junk, key_metadata in six.iteritems(targets_map_file['keys']): + if key_metadata['keytype'] in _SUPPORTED_KEY_TYPES: + # 'key_metadata' is stored in 'KEY_SCHEMA' format. Call + # create_from_metadata_format() to get the key in 'RSAKEY_SCHEMA' format, + # which is the format expected by 'add_key()'. Note: The 'keyids' + # returned by format_metadata_to_key() include keyids in addition to the + # default keyid listed in 'key_dict'. The additional keyids are + # generated according to securesystemslib.settings.HASH_ALGORITHMS. + + # The repo may have used hashing algorithms for the generated keyids that + # doesn't match the client's set of hash algorithms. Make sure to only + # used the repo's selected hashing algorithms. + hash_algorithms = securesystemslib.settings.HASH_ALGORITHMS + securesystemslib.settings.HASH_ALGORITHMS = key_metadata['keyid_hash_algorithms'] + key_dict, keyids = securesystemslib.keys.format_metadata_to_key(key_metadata) + securesystemslib.settings.HASH_ALGORITHMS = hash_algorithms + + try: + for keyid in keyids: + # Make sure to update key_dict['keyid'] to use one of the other valid + # keyids, otherwise add_key() will have no reference to it. + key_dict['keyid'] = keyid + add_key(key_dict, keyid=None, repository_name=repository_name) + + # Although keyid duplicates should *not* occur (unique dict keys), log a + # warning and continue. However, 'key_dict' may have already been + # adding to the keydb elsewhere. + except tuf.exceptions.KeyAlreadyExistsError as e: # pragma: no cover + logger.warning(e) + continue + + else: + logger.warning('Root Metadata file contains a key with an invalid keytype.') + # Iterate the keys found in 'root_metadata' by converting them to # 'RSAKEY_SCHEMA' if their type is 'rsa', and then adding them to the # key database using the provided keyid. diff --git a/tuf/roledb.py b/tuf/roledb.py index 37add72e3a..1e94e6baef 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -76,7 +76,7 @@ TOP_LEVEL_ROLES = ['root', 'targets', 'snapshot', 'timestamp'] -def create_roledb_from_root_metadata(root_metadata, repository_name='default'): +def create_roledb_from_root_metadata(root_metadata, repository_name='default', targets_map_file = None): """ Create a role database containing all of the unique roles found in @@ -140,6 +140,11 @@ def create_roledb_from_root_metadata(root_metadata, repository_name='default'): roleinfo['previous_keyids'] = roleinfo['keyids'] roleinfo['previous_threshold'] = roleinfo['threshold'] + if rolename == 'targets' and targets_map_file is not None: + roleinfo['keyids'] = [] + for keyid, _ in targets_map_file['keys']: + roleinfo['keyids'].append(keyid) + roleinfo['signatures'] = [] roleinfo['signing_keyids'] = [] roleinfo['partial_loaded'] = False From b2fe6e3e83288097299db7d5e63f5feb4b03266f Mon Sep 17 00:00:00 2001 From: marinamoore Date: Thu, 6 Aug 2020 15:08:33 -0700 Subject: [PATCH 2/9] Add initial TAP 13 tests The pr adds test targets mapping metadata using the existing delegated role from the test repository. It also adds an initial test for TAP 13 functionality Signed-off-by: marinamoore --- tests/repository_data/client/targets_map.json | 16 ++++++++++++++++ tests/test_updater.py | 17 +++++++++++++++++ tuf/roledb.py | 2 +- 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tests/repository_data/client/targets_map.json diff --git a/tests/repository_data/client/targets_map.json b/tests/repository_data/client/targets_map.json new file mode 100644 index 0000000000..47fd58351f --- /dev/null +++ b/tests/repository_data/client/targets_map.json @@ -0,0 +1,16 @@ +{ + "targets_filename": "role1", + "keys":{ + "c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a": { + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keytype": "ed25519", + "keyval": { + "public": "fcf224e55fa226056adf113ef1eb3d55e308b75b321c8c8316999d8c4fd9e0d9" + }, + "scheme": "ed25519" + } + } +} diff --git a/tests/test_updater.py b/tests/test_updater.py index 3ea0206adf..d2398692ce 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -1801,6 +1801,23 @@ def test_13__targets_of_role(self): targets=targets, skip_refresh=False) + def test_15__targets_map_file(self): + targets_map_file = os.path.join(self.client_directory, 'targets_map.json') + # Creating a repository instance using the targets map file. The test cases + # will use this client updater to refresh metadata, fetch target files, etc. + self.repository_updater = updater.Updater(self.repository_name, + self.repository_mirrors, targets_map_file) + + all_targets = self.repository_updater.all_targets() + self.assertEqual(len(all_targets), 1) + + + + + + + + class TestMultiRepoUpdater(unittest_toolbox.Modified_TestCase): diff --git a/tuf/roledb.py b/tuf/roledb.py index 1e94e6baef..edd899b0a7 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -142,7 +142,7 @@ def create_roledb_from_root_metadata(root_metadata, repository_name='default', t if rolename == 'targets' and targets_map_file is not None: roleinfo['keyids'] = [] - for keyid, _ in targets_map_file['keys']: + for keyid in targets_map_file['keys']: roleinfo['keyids'].append(keyid) roleinfo['signatures'] = [] From b47e4742de2107e2c9a9a82dc16b8339738a3877 Mon Sep 17 00:00:00 2001 From: marinamoore Date: Fri, 7 Aug 2020 09:45:23 -0700 Subject: [PATCH 3/9] Add tests for the targets map file Signed-off-by: marinamoore --- tests/test_updater.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/test_updater.py b/tests/test_updater.py index d2398692ce..743e97ba79 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -1802,19 +1802,32 @@ def test_13__targets_of_role(self): def test_15__targets_map_file(self): + # test badly formed targets map file + self.assertRaises(tuf.exceptions.Error, updater.Updater, self.repository_name, + self.repository_mirrors, 'no_file') + + targets_map_file = os.path.join(self.repository_directory, 'metadata', 'root.json') + self.assertRaises(securesystemslib.exceptions.FormatError, updater.Updater, + self.repository_name, self.repository_mirrors, targets_map_file) + + # set the correct map file targets_map_file = os.path.join(self.client_directory, 'targets_map.json') - # Creating a repository instance using the targets map file. The test cases - # will use this client updater to refresh metadata, fetch target files, etc. + + # Creating a repository instance using the targets map file. self.repository_updater = updater.Updater(self.repository_name, self.repository_mirrors, targets_map_file) + #ensure the map file was set + self.assertEqual('role1', self.repository_updater.targets_map_file['targets_filename']) + + # ensure that only targets in the targets map file are loaded all_targets = self.repository_updater.all_targets() self.assertEqual(len(all_targets), 1) - - - - + # ensure the targets map file replaces the targets role + self.repository_updater._load_metadata_from_file('current', 'targets') + self.assertRaises(tuf.exceptions.UnknownRoleError, + self.repository_updater._targets_of_role, 'role1') From 4ceda629924269224a7fbe9f2ba2d651fba501d0 Mon Sep 17 00:00:00 2001 From: marinamoore Date: Fri, 7 Aug 2020 11:29:49 -0700 Subject: [PATCH 4/9] This commit adds support for the targets map file with multiple repositories. This assumes that the targets map file will be the same on each repository listed in the map file. Signed-off-by: marinamoore --- tests/repository_data/targets_map.json | 16 ++++++++++++++++ tests/test_updater.py | 24 +++++++++++++++++++++++- tuf/client/updater.py | 12 ++++++++++-- 3 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 tests/repository_data/targets_map.json diff --git a/tests/repository_data/targets_map.json b/tests/repository_data/targets_map.json new file mode 100644 index 0000000000..47fd58351f --- /dev/null +++ b/tests/repository_data/targets_map.json @@ -0,0 +1,16 @@ +{ + "targets_filename": "role1", + "keys":{ + "c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a": { + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keytype": "ed25519", + "keyval": { + "public": "fcf224e55fa226056adf113ef1eb3d55e308b75b321c8c8316999d8c4fd9e0d9" + }, + "scheme": "ed25519" + } + } +} diff --git a/tests/test_updater.py b/tests/test_updater.py index 743e97ba79..7832d180e4 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -1813,7 +1813,7 @@ def test_15__targets_map_file(self): # set the correct map file targets_map_file = os.path.join(self.client_directory, 'targets_map.json') - # Creating a repository instance using the targets map file. + # Creating a repository instance using the targets map file. self.repository_updater = updater.Updater(self.repository_name, self.repository_mirrors, targets_map_file) @@ -1860,6 +1860,7 @@ def setUp(self): original_client = os.path.join(original_repository_files, 'client', 'test_repository1') original_keystore = os.path.join(original_repository_files, 'keystore') original_map_file = os.path.join(original_repository_files, 'map.json') + original_targets_map_file = os.path.join(original_repository_files, 'targets_map.json') # Save references to the often-needed client repository directories. # Test cases need these references to access metadata and target files. @@ -1884,6 +1885,7 @@ def setUp(self): 'keystore') self.map_file = os.path.join(self.client_directory, 'map.json') self.map_file2 = os.path.join(self.client_directory2, 'map.json') + self.targets_map_file = os.path.join(self.temporary_repository_root, 'targets_map_file') # Copy the original 'repository', 'client', and 'keystore' directories # to the temporary repository the test cases can use. @@ -1894,6 +1896,7 @@ def setUp(self): shutil.copyfile(original_map_file, self.map_file) shutil.copyfile(original_map_file, self.map_file2) shutil.copytree(original_keystore, self.keystore_directory) + shutil.copyfile(original_targets_map_file, self.targets_map_file) # Launch a SimpleHTTPServer (serves files in the current directory). # Test cases will request metadata and target files that have been @@ -2152,6 +2155,25 @@ def test_get_updater(self): + + + def test_targets_map_file(self): + map_file = os.path.join(self.client_directory, 'map.json') + multi_repo_updater = updater.MultiRepoUpdater(map_file, self.targets_map_file) + + #self.assertTrue('targets_map.json' in multi_repo_updater.targets_map_filename) + + print(multi_repo_updater.map_file) + print(multi_repo_updater.repository_names_to_mirrors) + print(multi_repo_updater.repository_names_to_updaters) + + # Does the repository use the targets map file? + repository_updater = multi_repo_updater.get_updater('test_repository2') + + self.assertEqual(repository_updater.targets_map_file['targets_filename'], 'role1') + + + def _load_role_keys(keystore_directory): # Populating 'self.role_keys' by importing the required public and private diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 57da6ead55..a63c9d4e6a 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -199,7 +199,7 @@ class MultiRepoUpdater(object): None. """ - def __init__(self, map_file): + def __init__(self, map_file, targets_map_filename=None): # Is 'map_file' a path? If not, raise # 'securesystemslib.exceptions.FormatError'. The actual content of the map # file is validated later on in this method. @@ -228,6 +228,14 @@ def __init__(self, map_file): # } self.repository_names_to_mirrors = self.map_file['repositories'] + self.targets_map_filename = None + + if targets_map_filename is not None: + # Is 'targets_map_file' a path? If not, raise + # 'securesystemslib.exceptions.FormatError'. The actual content of the map + # file is validated later on in this method. + securesystemslib.formats.PATH_SCHEMA.check_match(targets_map_filename) + self.targets_map_filename = targets_map_filename def get_valid_targetinfo(self, target_filename, match_custom_field=True): @@ -517,7 +525,7 @@ def get_updater(self, repository_name): # NOTE: State (e.g., keys) should NOT be shared across different # updater instances. logger.debug('Adding updater for ' + repr(repository_name)) - updater = tuf.client.updater.Updater(repository_name, mirrors) + updater = tuf.client.updater.Updater(repository_name, mirrors, self.targets_map_filename) except Exception: return None From 9b6f0bc195846b80f77086d59b5a194aa77dd392 Mon Sep 17 00:00:00 2001 From: marinamoore Date: Fri, 7 Aug 2020 11:50:03 -0700 Subject: [PATCH 5/9] Update docstrings for targets map file Signed-off-by: marinamoore --- tests/test_updater.py | 6 ------ tuf/client/updater.py | 11 +++++++++++ tuf/keydb.py | 5 +++++ tuf/roledb.py | 4 ++++ 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/test_updater.py b/tests/test_updater.py index 7832d180e4..54e614aa6a 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -2161,12 +2161,6 @@ def test_targets_map_file(self): map_file = os.path.join(self.client_directory, 'map.json') multi_repo_updater = updater.MultiRepoUpdater(map_file, self.targets_map_file) - #self.assertTrue('targets_map.json' in multi_repo_updater.targets_map_filename) - - print(multi_repo_updater.map_file) - print(multi_repo_updater.repository_names_to_mirrors) - print(multi_repo_updater.repository_names_to_updaters) - # Does the repository use the targets map file? repository_updater = multi_repo_updater.get_updater('test_repository2') diff --git a/tuf/client/updater.py b/tuf/client/updater.py index a63c9d4e6a..1cc39eff55 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -186,6 +186,10 @@ class MultiRepoUpdater(object): The path of the map file. The map file is needed to determine which repositories to query given a target file. + targets_map_filename: + The path of the targets map file. This targets map file + will be used by all repositories. + securesystemslib.exceptions.FormatError, if the map file is improperly formatted. @@ -595,6 +599,9 @@ class Updater(object): self.repository_name: The name of the updater instance. + self.targets_map_file: + The contents of the targets map file. + refresh(): This method downloads, verifies, and loads metadata for the top-level @@ -673,6 +680,10 @@ def __init__(self, repository_name, repository_mirrors, targets_map_file = None) 'metadata_path': 'metadata', 'targets_path': 'targets', 'confined_target_dirs': ['']}} + targets_map_file: + The name of the targets map file. If one is not provided, + self.targets_map_file will be set to None and the top-level + targets metadata from the repository will be used. securesystemslib.exceptions.FormatError: diff --git a/tuf/keydb.py b/tuf/keydb.py index 790a81c4d6..0c90c7a693 100755 --- a/tuf/keydb.py +++ b/tuf/keydb.py @@ -79,6 +79,11 @@ def create_keydb_from_root_metadata(root_metadata, repository_name='default', ta The name of the repository to store the key information. If not supplied, the key database is populated for the 'default' repository. + targets_map_file: + The contents of the targets map file. The keys from the targets map file + will be added to the keydb. If not provided the keys from the default + targets files will be used. + securesystemslib.exceptions.FormatError, if 'root_metadata' does not have the correct format. diff --git a/tuf/roledb.py b/tuf/roledb.py index edd899b0a7..29cc2e40f2 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -92,6 +92,10 @@ def create_roledb_from_root_metadata(root_metadata, repository_name='default', t The name of the repository to store 'root_metadata'. If not supplied, 'rolename' is added to the 'default' repository. + targets_map_file: + The contents of the targets map file. This will be used to add the keys + to the targets role. If not provided, the default targets keys will be used. + securesystemslib.exceptions.FormatError, if 'root_metadata' does not have the correct object format. From 4238496a24980bf192bbd70751374fb3cdac0b6e Mon Sep 17 00:00:00 2001 From: marinamoore Date: Thu, 20 Aug 2020 13:51:49 -0700 Subject: [PATCH 6/9] Fix style issues and remove repeated code as discussed in the pull request. This commit also adds some additional comments. Signed-off-by: marinamoore --- tuf/client/updater.py | 35 +++++++++++++++-------------- tuf/keydb.py | 51 ++++++++++--------------------------------- tuf/roledb.py | 5 +++-- 3 files changed, 34 insertions(+), 57 deletions(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 1cc39eff55..e46e2d52c7 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -232,14 +232,13 @@ def __init__(self, map_file, targets_map_filename=None): # } self.repository_names_to_mirrors = self.map_file['repositories'] - self.targets_map_filename = None - + # If there is a targets_map_file, set self.targets_map_filename. + # This map file will be used with all repositories in place of the + # top level targets file. if targets_map_filename is not None: - # Is 'targets_map_file' a path? If not, raise - # 'securesystemslib.exceptions.FormatError'. The actual content of the map - # file is validated later on in this method. securesystemslib.formats.PATH_SCHEMA.check_match(targets_map_filename) - self.targets_map_filename = targets_map_filename + + self.targets_map_filename = targets_map_filename def get_valid_targetinfo(self, target_filename, match_custom_field=True): @@ -641,7 +640,8 @@ class Updater(object): http://www.python.org/dev/peps/pep-0008/#method-names-and-instance-variables """ - def __init__(self, repository_name, repository_mirrors, targets_map_file = None): + def __init__(self, repository_name, repository_mirrors, + targets_map_filename=None): """ Constructor. Instantiating an updater object causes all the metadata @@ -680,7 +680,7 @@ def __init__(self, repository_name, repository_mirrors, targets_map_file = None) 'metadata_path': 'metadata', 'targets_path': 'targets', 'confined_target_dirs': ['']}} - targets_map_file: + targets_map_filename: The name of the targets map file. If one is not provided, self.targets_map_file will be set to None and the top-level targets metadata from the repository will be used. @@ -715,14 +715,13 @@ def __init__(self, repository_name, repository_mirrors, targets_map_file = None) self.mirrors = repository_mirrors self.targets_map_file = None - if targets_map_file is not None: - # Is 'targets_map_file' a path? If not, raise - # 'securesystemslib.exceptions.FormatError'. The actual content of the map - # file is validated later on in this method. - securesystemslib.formats.PATH_SCHEMA.check_match(targets_map_file) + # If targets_map_filename is provided, load the targets_map_file. + if targets_map_filename is not None: + securesystemslib.formats.PATH_SCHEMA.check_match(targets_map_filename) try: - self.targets_map_file = securesystemslib.util.load_json_file(targets_map_file) + self.targets_map_file = securesystemslib.util.load_json_file( + targets_map_filename) except (securesystemslib.exceptions.Error) as e: raise tuf.exceptions.Error('Cannot load the targets map file: ' + str(e)) @@ -859,7 +858,9 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): # Save and construct the full metadata path. metadata_directory = self.metadata_directory[metadata_set] # For top-level targets, the targets map file may overwrite the - # targets metadata on the repository + # targets metadata on the repository. If this is the case, ignore + # the 'targets.json' file on the repository and instead load + # the targets metadata indicated by the targets map file. if metadata_role == 'targets' and self.targets_map_file is not None: metadata_filename = self.targets_map_file['targets_filename'] + '.json' else: @@ -913,7 +914,9 @@ def _rebuild_key_and_role_db(self): This private method is called when a new/updated 'root' metadata file is loaded or when updater.refresh() is called. This method will only store the role information of the top-level roles (i.e., 'root', 'targets', - 'snapshot', 'timestamp'). + 'snapshot', 'timestamp'). If a targets map file is used, this will + additionally load the key and role information from the targets map + file. None. diff --git a/tuf/keydb.py b/tuf/keydb.py index 0c90c7a693..48a619c25d 100755 --- a/tuf/keydb.py +++ b/tuf/keydb.py @@ -60,7 +60,8 @@ _keydb_dict['default'] = {} -def create_keydb_from_root_metadata(root_metadata, repository_name='default', targets_map_file = None): +def create_keydb_from_root_metadata(root_metadata, repository_name='default', + targets_map_file=None): """ Populate the key database with the unique keys found in 'root_metadata'. @@ -116,48 +117,20 @@ def create_keydb_from_root_metadata(root_metadata, repository_name='default', ta else: create_keydb(repository_name) + keys_to_add = root_metadata['keys'] if targets_map_file is not None: tuf.formats.TARGETS_MAPFILE_SCHEMA.check_match(targets_map_file) - for junk, key_metadata in six.iteritems(targets_map_file['keys']): - if key_metadata['keytype'] in _SUPPORTED_KEY_TYPES: - # 'key_metadata' is stored in 'KEY_SCHEMA' format. Call - # create_from_metadata_format() to get the key in 'RSAKEY_SCHEMA' format, - # which is the format expected by 'add_key()'. Note: The 'keyids' - # returned by format_metadata_to_key() include keyids in addition to the - # default keyid listed in 'key_dict'. The additional keyids are - # generated according to securesystemslib.settings.HASH_ALGORITHMS. - - # The repo may have used hashing algorithms for the generated keyids that - # doesn't match the client's set of hash algorithms. Make sure to only - # used the repo's selected hashing algorithms. - hash_algorithms = securesystemslib.settings.HASH_ALGORITHMS - securesystemslib.settings.HASH_ALGORITHMS = key_metadata['keyid_hash_algorithms'] - key_dict, keyids = securesystemslib.keys.format_metadata_to_key(key_metadata) - securesystemslib.settings.HASH_ALGORITHMS = hash_algorithms - - try: - for keyid in keyids: - # Make sure to update key_dict['keyid'] to use one of the other valid - # keyids, otherwise add_key() will have no reference to it. - key_dict['keyid'] = keyid - add_key(key_dict, keyid=None, repository_name=repository_name) - - # Although keyid duplicates should *not* occur (unique dict keys), log a - # warning and continue. However, 'key_dict' may have already been - # adding to the keydb elsewhere. - except tuf.exceptions.KeyAlreadyExistsError as e: # pragma: no cover - logger.warning(e) - continue - - else: - logger.warning('Root Metadata file contains a key with an invalid keytype.') - - # Iterate the keys found in 'root_metadata' by converting them to - # 'RSAKEY_SCHEMA' if their type is 'rsa', and then adding them to the - # key database using the provided keyid. - for keyid, key_metadata in six.iteritems(root_metadata['keys']): + # If any keyids from root metadata match those from targets_map_metadata, + # only the root metadata keys will be added. Add all keys to keys_to_add + keys_to_add = targets_map_file['keys'].copy() + keys_to_add.update(root_metadata['keys']) + + # Iterate the keys found in 'root_metadata' and 'targets_map_file' by + # converting them to 'RSAKEY_SCHEMA' if their type is 'rsa', and then adding + # them to the key database using the provided keyid. + for keyid, key_metadata in six.iteritems(keys_to_add): if key_metadata['keytype'] in _SUPPORTED_KEY_TYPES: # 'key_metadata' is stored in 'KEY_SCHEMA' format. Call # create_from_metadata_format() to get the key in 'RSAKEY_SCHEMA' format, diff --git a/tuf/roledb.py b/tuf/roledb.py index 29cc2e40f2..7dffc8f7fa 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -76,7 +76,8 @@ TOP_LEVEL_ROLES = ['root', 'targets', 'snapshot', 'timestamp'] -def create_roledb_from_root_metadata(root_metadata, repository_name='default', targets_map_file = None): +def create_roledb_from_root_metadata(root_metadata, repository_name='default', + targets_map_file=None): """ Create a role database containing all of the unique roles found in @@ -144,7 +145,7 @@ def create_roledb_from_root_metadata(root_metadata, repository_name='default', t roleinfo['previous_keyids'] = roleinfo['keyids'] roleinfo['previous_threshold'] = roleinfo['threshold'] - if rolename == 'targets' and targets_map_file is not None: + elif rolename == 'targets' and targets_map_file is not None: roleinfo['keyids'] = [] for keyid in targets_map_file['keys']: roleinfo['keyids'].append(keyid) From dce7e4a7a505c836d72266ed6c88b601bbe38e08 Mon Sep 17 00:00:00 2001 From: marinamoore Date: Mon, 31 Aug 2020 14:11:46 -0700 Subject: [PATCH 7/9] Use targets map metadata in _update_metadata_if_changed If targets map is used, use the referenced metadata instead of the top level metadata. This makes _update_metadata_if_changed consistent with _update_metadata. Signed-off-by: marinamoore --- tuf/client/updater.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index e46e2d52c7..04a2f4fe46 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -1967,7 +1967,10 @@ def _update_metadata_if_changed(self, metadata_role, None. """ - metadata_filename = metadata_role + '.json' + if self.targets_map_file is not None and metadata_role == 'targets': + metadata_filename = self.targets_map_file['targets_filename'] + '.json' + else: + metadata_filename = metadata_role + '.json' expected_versioninfo = None # Ensure the referenced metadata has been loaded. The 'root' role may be From 8eb0c5d396139f9e523327681ecedada8f3ee842 Mon Sep 17 00:00:00 2001 From: marinamoore Date: Mon, 21 Sep 2020 11:56:07 -0700 Subject: [PATCH 8/9] Document the exception when the target map file is not loaded. If the targets map file cannot be loaded using load_json_file, a tuf.exceptions.Error exception will be raised. Signed-off-by: marinamoore --- tuf/client/updater.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 04a2f4fe46..7330fed409 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -693,6 +693,11 @@ def __init__(self, repository_name, repository_mirrors, If there is an error with the updater's repository files, such as a missing 'root.json' file. + tuf.exceptions.Error: + If the targets map file cannot be loaded. This may be due to a + securesystemslib.exceptions.Error if the targets map filename + cannot be parsed, or and IOError in the case of runtime exceptions. + Th metadata files (e.g., 'root.json', 'targets.json') for the top- level roles are read from disk and stored in dictionaries. In addition, the From a6d6cdcb9ca9fe5abc05f66a9cfc27fd76f7a244 Mon Sep 17 00:00:00 2001 From: Marina Moore Date: Fri, 10 Dec 2021 14:36:30 -0500 Subject: [PATCH 9/9] filename -> rolename Signed-off-by: Marina Moore --- tests/repository_data/client/targets_map.json | 2 +- tests/repository_data/targets_map.json | 2 +- tests/test_updater.py | 4 ++-- tuf/client/updater.py | 6 +++--- tuf/formats.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/repository_data/client/targets_map.json b/tests/repository_data/client/targets_map.json index 47fd58351f..859e4e6e69 100644 --- a/tests/repository_data/client/targets_map.json +++ b/tests/repository_data/client/targets_map.json @@ -1,5 +1,5 @@ { - "targets_filename": "role1", + "targets_rolename": "role1", "keys":{ "c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a": { "keyid_hash_algorithms": [ diff --git a/tests/repository_data/targets_map.json b/tests/repository_data/targets_map.json index 47fd58351f..859e4e6e69 100644 --- a/tests/repository_data/targets_map.json +++ b/tests/repository_data/targets_map.json @@ -1,5 +1,5 @@ { - "targets_filename": "role1", + "targets_rolename": "role1", "keys":{ "c8022fa1e9b9cb239a6b362bbdffa9649e61ad2cb699d2e4bc4fdf7930a0e64a": { "keyid_hash_algorithms": [ diff --git a/tests/test_updater.py b/tests/test_updater.py index 54e614aa6a..df09c6646b 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -1818,7 +1818,7 @@ def test_15__targets_map_file(self): self.repository_mirrors, targets_map_file) #ensure the map file was set - self.assertEqual('role1', self.repository_updater.targets_map_file['targets_filename']) + self.assertEqual('role1', self.repository_updater.targets_map_file['targets_rolename']) # ensure that only targets in the targets map file are loaded all_targets = self.repository_updater.all_targets() @@ -2164,7 +2164,7 @@ def test_targets_map_file(self): # Does the repository use the targets map file? repository_updater = multi_repo_updater.get_updater('test_repository2') - self.assertEqual(repository_updater.targets_map_file['targets_filename'], 'role1') + self.assertEqual(repository_updater.targets_map_file['targets_rolename'], 'role1') diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 7330fed409..15a428b1a9 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -867,7 +867,7 @@ def _load_metadata_from_file(self, metadata_set, metadata_role): # the 'targets.json' file on the repository and instead load # the targets metadata indicated by the targets map file. if metadata_role == 'targets' and self.targets_map_file is not None: - metadata_filename = self.targets_map_file['targets_filename'] + '.json' + metadata_filename = self.targets_map_file['targets_rolename'] + '.json' else: metadata_filename = metadata_role + '.json' metadata_filepath = os.path.join(metadata_directory, metadata_filename) @@ -1835,7 +1835,7 @@ def _update_metadata(self, metadata_role, upperbound_filelength, version=None): # Construct the metadata filename as expected by the download/mirror # modules. if self.targets_map_file is not None and metadata_role == 'targets': - metadata_filename = self.targets_map_file['targets_filename'] + '.json' + metadata_filename = self.targets_map_file['targets_rolename'] + '.json' else: metadata_filename = metadata_role + '.json' @@ -1973,7 +1973,7 @@ def _update_metadata_if_changed(self, metadata_role, """ if self.targets_map_file is not None and metadata_role == 'targets': - metadata_filename = self.targets_map_file['targets_filename'] + '.json' + metadata_filename = self.targets_map_file['targets_rolename'] + '.json' else: metadata_filename = metadata_role + '.json' expected_versioninfo = None diff --git a/tuf/formats.py b/tuf/formats.py index 082912bc22..7979dd3d59 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -279,7 +279,7 @@ # An object containing the targets map file. The format of the targets # map file is covered in TAP 13 TARGETS_MAPFILE_SCHEMA = SCHEMA.Object( - targets_filename = NAME_SCHEMA, + targets_rolename = NAME_SCHEMA, keys = KEYDICT_SCHEMA) # An object containing the map file's "mapping" attribute.