From 98ddbdd3978a33805f918e8a24e5bb36a25f3d2a Mon Sep 17 00:00:00 2001 From: Teodora Sechkova Date: Thu, 18 Mar 2021 18:11:20 +0200 Subject: [PATCH 01/17] Add .gitattributes file For compatibility with Windows systems, declare repository_data files to always have LF line endings on checkout. A trailing "/**" matches everything inside, with infinite depth. Signed-off-by: Teodora Sechkova --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..66709ac428 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Files that will always have LF line endings on checkout. +tests/repository_data/** text eol=lf + From 901b7f4491fcfe15bd1dab1c4353b75cb0da59f1 Mon Sep 17 00:00:00 2001 From: Martin Vrachev Date: Wed, 24 Mar 2021 15:38:11 +0200 Subject: [PATCH 02/17] Remove additional version settup in Signed Signed-off-by: Martin Vrachev --- tuf/api/metadata.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tuf/api/metadata.py b/tuf/api/metadata.py index 4d236b576b..d928c5afb0 100644 --- a/tuf/api/metadata.py +++ b/tuf/api/metadata.py @@ -311,7 +311,6 @@ def __init__( ) -> None: self._type = _type - self.version = version self.spec_version = spec_version self.expires = expires From cbc814ffa8fcdff07cbf4b09d4f0070d58323898 Mon Sep 17 00:00:00 2001 From: Martin Vrachev Date: Wed, 24 Mar 2021 15:55:29 +0200 Subject: [PATCH 03/17] New API: Fix exception message for version Signed-off-by: Martin Vrachev --- tuf/api/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tuf/api/metadata.py b/tuf/api/metadata.py index 4d236b576b..4d44bc11a5 100644 --- a/tuf/api/metadata.py +++ b/tuf/api/metadata.py @@ -317,7 +317,7 @@ def __init__( # TODO: Should we separate data validation from constructor? if version < 0: - raise ValueError(f"version must be < 0, got {version}") + raise ValueError(f"version must be >= 0, got {version}") self.version = version @staticmethod From fd515dfd3e1b9f9f1bf81ac3caeb3558abcdc1ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Mar 2021 10:04:15 +0000 Subject: [PATCH 04/17] build(deps): bump cryptography from 3.4.6 to 3.4.7 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.6 to 3.4.7. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.4.6...3.4.7) Signed-off-by: dependabot[bot] --- requirements-pinned.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-pinned.txt b/requirements-pinned.txt index 7a02787905..c31b5adc5b 100644 --- a/requirements-pinned.txt +++ b/requirements-pinned.txt @@ -1,7 +1,7 @@ certifi==2020.12.5 # via requests cffi==1.14.5 # via cryptography, pynacl chardet==4.0.0 # via requests -cryptography==3.4.6 # via securesystemslib +cryptography==3.4.7 # via securesystemslib idna==2.10 # via requests pycparser==2.20 # via cffi pynacl==1.4.0 # via securesystemslib From 7e23201741537215b14f94f6012d64f9056aa573 Mon Sep 17 00:00:00 2001 From: Philippe Coval Date: Sat, 27 Mar 2021 10:34:12 +0000 Subject: [PATCH 05/17] build: Add keys to sdist archive for tests While packaging latest release for debian from pypi tarball I noticed some tests' files were not shipped, let's add them. Relate-to: https://github.com/theupdateframework/tuf/issues/263 Signed-off-by: Philippe Coval --- MANIFEST.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 0fd516697a..4161a92598 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,10 +2,12 @@ include LICENSE* include README.md include tox.ini include tests/repository_data/keystore/delegation_key -include tests/repository_data/keystore/root_key +include tests/repository_data/keystore/root_key* include tests/repository_data/keystore/snapshot_key include tests/repository_data/keystore/targets_key include tests/repository_data/keystore/timestamp_key +include tests/ssl_certs/*.crt +include tests/ssl_certs/*.key recursive-include docs *.txt recursive-include docs *.md From 5a0047db89ed5ba8e7b5418048caa69b0474648f Mon Sep 17 00:00:00 2001 From: Philippe Coval Date: Wed, 31 Mar 2021 15:05:53 +0200 Subject: [PATCH 06/17] test: Start coverage from python module Runing module from python will help portability between systems. Those scripts may be renamed by distros, for instance Debian provides python3-converage. Signed-off-by: Philippe Coval --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 9cdafa6e58..77bc09e359 100644 --- a/tox.ini +++ b/tox.ini @@ -15,8 +15,8 @@ changedir = tests commands = python --version - coverage run aggregate_tests.py - coverage report -m --fail-under 97 + python -m coverage run aggregate_tests.py + python -m coverage report -m --fail-under 97 deps = -r{toxinidir}/requirements-test.txt @@ -35,8 +35,8 @@ deps = --editable {toxinidir} commands = - coverage run aggregate_tests.py - coverage report -m + python -m coverage run aggregate_tests.py + python -m coverage report -m [testenv:lint] commands = From 420937a96d19ae0aa6681282e57bb78f364c609b Mon Sep 17 00:00:00 2001 From: maoyangLiu Date: Thu, 1 Apr 2021 20:37:27 +0800 Subject: [PATCH 07/17] fix the outdate url Signed-off-by: maoyangLiu --- docs/tuf-spec.0.9.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tuf-spec.0.9.txt b/docs/tuf-spec.0.9.txt index 1140747e2b..e5bdeb8368 100644 --- a/docs/tuf-spec.0.9.txt +++ b/docs/tuf-spec.0.9.txt @@ -1 +1 @@ -The TUF specification file has been moved to https://github.com/theupdateframework/specification/blob/master/historical/tuf-spec.md +The TUF specification file has been moved to https://github.com/theupdateframework/specification/blob/master/historical/tuf-spec.0.9.txt From d8b3554662c5fd1e4c714443582a81c457eacb3c Mon Sep 17 00:00:00 2001 From: Velichka Atanasova Date: Thu, 8 Apr 2021 14:24:54 +0300 Subject: [PATCH 08/17] Remove use of six Remove use of six Signed-off-by: Velichka Atanasova Replace the use of dict.items(mydict) with mydict.items(), dict.keys(mydict) with mydict.keys() and dict.values(mydict) with mydict.values() Signed-off-by: Velichka Atanasova Replace 'import urllib' and 'import urllib.x' with 'from urllib import x' for vendor compatibility Signed-off-by: Velichka Atanasova --- pylintrc | 4 +-- requirements-pinned.txt | 1 - requirements.txt | 1 - setup.py | 3 +-- tests/simple_https_server.py | 6 ++--- tests/simple_server.py | 9 +++---- tests/slow_retrieval_server.py | 7 +++--- tests/test_arbitrary_package_attack.py | 6 ++--- tests/test_endless_data_attack.py | 8 +++--- tests/test_extraneous_dependencies_attack.py | 3 +-- tests/test_formats.py | 9 +++---- tests/test_indefinite_freeze_attack.py | 8 +++--- tests/test_key_revocation_integration.py | 1 - tests/test_log.py | 4 +-- tests/test_mix_and_match_attack.py | 3 +-- .../test_multiple_repositories_integration.py | 3 +-- tests/test_replay_attack.py | 9 ++++--- tests/test_repository_lib.py | 3 +-- tests/test_slow_retrieval_attack.py | 4 +-- tests/test_updater.py | 15 ++++++----- .../test_updater_root_rotation_integration.py | 13 +++++----- tuf/client/updater.py | 20 +++++++-------- tuf/developer_tool.py | 3 +-- tuf/download.py | 4 +-- tuf/exceptions.py | 6 ++--- tuf/formats.py | 16 +++++------- tuf/keydb.py | 6 ++--- tuf/mirrors.py | 8 +++--- tuf/repository_lib.py | 25 +++++++++---------- tuf/repository_tool.py | 3 +-- tuf/requests_fetcher.py | 4 +-- tuf/roledb.py | 4 +-- tuf/scripts/repo.py | 7 +++--- 33 files changed, 99 insertions(+), 127 deletions(-) diff --git a/pylintrc b/pylintrc index 8f3e9f4b5a..402406d245 100644 --- a/pylintrc +++ b/pylintrc @@ -287,7 +287,7 @@ ignore-on-opaque-inference=yes # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local, six.moves +ignored-classes=optparse.Values,thread._local,_thread._local # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime @@ -334,7 +334,7 @@ init-import=no # List of qualified module names which can have objects that can redefine # builtins. -redefining-builtins-modules=six.moves,future.builtins +redefining-builtins-modules=future.builtins [CLASSES] diff --git a/requirements-pinned.txt b/requirements-pinned.txt index c31b5adc5b..e1e8a4de75 100644 --- a/requirements-pinned.txt +++ b/requirements-pinned.txt @@ -7,5 +7,4 @@ pycparser==2.20 # via cffi pynacl==1.4.0 # via securesystemslib requests==2.25.1 securesystemslib[crypto,pynacl]==0.20.0 -six==1.15.0 urllib3==1.26.4 # via requests diff --git a/requirements.txt b/requirements.txt index 0ee7a97951..ed3328659f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,4 +43,3 @@ # securesystemslib[crypto, pynacl] requests -six diff --git a/setup.py b/setup.py index 10aa219248..0b5b35592a 100755 --- a/setup.py +++ b/setup.py @@ -113,8 +113,7 @@ python_requires="~=3.6", install_requires = [ 'requests>=2.19.1', - 'securesystemslib>=0.20.0', - 'six>=1.11.0' + 'securesystemslib>=0.20.0' ], packages = find_packages(exclude=['tests']), scripts = [ diff --git a/tests/simple_https_server.py b/tests/simple_https_server.py index 61c595739f..ad54a4626b 100755 --- a/tests/simple_https_server.py +++ b/tests/simple_https_server.py @@ -40,7 +40,7 @@ import sys import ssl import os -import six +import http.server keyfile = os.path.join('ssl_certs', 'ssl_cert.key') certfile = os.path.join('ssl_certs', 'ssl_cert.crt') @@ -49,8 +49,8 @@ if len(sys.argv) > 1 and os.path.exists(sys.argv[1]): certfile = sys.argv[1] -httpd = six.moves.BaseHTTPServer.HTTPServer(('localhost', 0), - six.moves.SimpleHTTPServer.SimpleHTTPRequestHandler) +httpd = http.server.HTTPServer(('localhost', 0), + http.server.SimpleHTTPRequestHandler) httpd.socket = ssl.wrap_socket( httpd.socket, keyfile=keyfile, certfile=certfile, server_side=True) diff --git a/tests/simple_server.py b/tests/simple_server.py index bec8e7b07f..cbdcced5d3 100755 --- a/tests/simple_server.py +++ b/tests/simple_server.py @@ -35,9 +35,8 @@ import sys import random - -import six -from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler +import socketserver +from http.server import SimpleHTTPRequestHandler class QuietHTTPRequestHandler(SimpleHTTPRequestHandler): @@ -64,9 +63,9 @@ def log_request(self, code='-', size='-'): # Allow re-use so you can re-run tests as often as you want even if the # tests re-use ports. Otherwise TCP TIME-WAIT prevents reuse for ~1 minute -six.moves.socketserver.TCPServer.allow_reuse_address = True +socketserver.TCPServer.allow_reuse_address = True -httpd = six.moves.socketserver.TCPServer(('localhost', 0), handler) +httpd = socketserver.TCPServer(('localhost', 0), handler) port_message = 'bind succeeded, server port is: ' \ + str(httpd.server_address[1]) print(port_message) diff --git a/tests/slow_retrieval_server.py b/tests/slow_retrieval_server.py index 7740d73b88..44c764ba67 100755 --- a/tests/slow_retrieval_server.py +++ b/tests/slow_retrieval_server.py @@ -32,13 +32,12 @@ import os import sys import time - -import six +import http.server # HTTP request handler. -class Handler(six.moves.BaseHTTPServer.BaseHTTPRequestHandler): +class Handler(http.server.BaseHTTPRequestHandler): # Overwrite do_GET. def do_GET(self): @@ -66,7 +65,7 @@ def do_GET(self): if __name__ == '__main__': server_address = ('localhost', 0) - httpd = six.moves.BaseHTTPServer.HTTPServer(server_address, Handler) + httpd = http.server.HTTPServer(server_address, Handler) port_message = 'bind succeeded, server port is: ' \ + str(httpd.server_address[1]) print(port_message) diff --git a/tests/test_arbitrary_package_attack.py b/tests/test_arbitrary_package_attack.py index 21096fb4bf..b31ffe3167 100755 --- a/tests/test_arbitrary_package_attack.py +++ b/tests/test_arbitrary_package_attack.py @@ -43,6 +43,7 @@ import logging import unittest import sys +from urllib import request import tuf import tuf.formats @@ -55,7 +56,6 @@ from tests import utils import securesystemslib -import six logger = logging.getLogger(__name__) @@ -174,7 +174,7 @@ def test_without_tuf(self): url_file = os.path.join(url_prefix, 'targets', 'file1.txt') # On Windows, the URL portion should not contain back slashes. - six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_target_path) + request.urlretrieve(url_file.replace('\\', '/'), client_target_path) self.assertTrue(os.path.exists(client_target_path)) length, hashes = securesystemslib.util.get_file_details(client_target_path) @@ -188,7 +188,7 @@ def test_without_tuf(self): malicious_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes) # On Windows, the URL portion should not contain back slashes. - six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_target_path) + request.urlretrieve(url_file.replace('\\', '/'), client_target_path) length, hashes = securesystemslib.util.get_file_details(client_target_path) download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes) diff --git a/tests/test_endless_data_attack.py b/tests/test_endless_data_attack.py index 01701d9830..4cc4e9370a 100755 --- a/tests/test_endless_data_attack.py +++ b/tests/test_endless_data_attack.py @@ -46,6 +46,7 @@ import logging import unittest import sys +from urllib import request import tuf import tuf.formats @@ -57,7 +58,6 @@ from tests import utils import securesystemslib -import six logger = logging.getLogger(__name__) @@ -176,7 +176,7 @@ def test_without_tuf(self): url_file = os.path.join(url_prefix, 'targets', 'file1.txt') # On Windows, the URL portion should not contain backslashes. - six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_target_path) + request.urlretrieve(url_file.replace('\\', '/'), client_target_path) self.assertTrue(os.path.exists(client_target_path)) length, hashes = securesystemslib.util.get_file_details(client_target_path) @@ -194,7 +194,7 @@ def test_without_tuf(self): self.assertTrue(large_length > length) # On Windows, the URL portion should not contain backslashes. - six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_target_path) + request.urlretrieve(url_file.replace('\\', '/'), client_target_path) length, hashes = securesystemslib.util.get_file_details(client_target_path) download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes) @@ -269,7 +269,7 @@ def test_with_tuf(self): self.repository_updater.refresh() except tuf.exceptions.NoWorkingMirrorError as exception: - for mirror_url, mirror_error in six.iteritems(exception.mirror_errors): + for mirror_url, mirror_error in exception.mirror_errors.items(): self.assertTrue(isinstance(mirror_error, securesystemslib.exceptions.Error)) else: diff --git a/tests/test_extraneous_dependencies_attack.py b/tests/test_extraneous_dependencies_attack.py index 2e71ee800b..a75ca904d4 100755 --- a/tests/test_extraneous_dependencies_attack.py +++ b/tests/test_extraneous_dependencies_attack.py @@ -60,7 +60,6 @@ from tests import utils import securesystemslib -import six logger = logging.getLogger(__name__) @@ -206,7 +205,7 @@ def test_with_tuf(self): # Verify that the specific 'tuf.exceptions.ForbiddenTargetError' exception is raised # by each mirror. except tuf.exceptions.NoWorkingMirrorError as exception: - for mirror_url, mirror_error in six.iteritems(exception.mirror_errors): + for mirror_url, mirror_error in exception.mirror_errors.items(): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'metadata', 'role1.json') diff --git a/tests/test_formats.py b/tests/test_formats.py index d34c2363cf..5a657689fd 100755 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -40,7 +40,6 @@ import securesystemslib import securesystemslib.util -import six class TestFormats(unittest.TestCase): @@ -287,7 +286,7 @@ def test_schemas(self): # Iterate 'valid_schemas', ensuring each 'valid_schema' correctly matches # its respective 'schema_type'. - for schema_name, (schema_type, valid_schema) in six.iteritems(valid_schemas): + for schema_name, (schema_type, valid_schema) in valid_schemas.items(): if not schema_type.matches(valid_schema): print('bad schema: ' + repr(valid_schema)) self.assertEqual(True, schema_type.matches(valid_schema)) @@ -295,7 +294,7 @@ def test_schemas(self): # Test conditions for invalid schemas. # Set the 'valid_schema' of 'valid_schemas' to an invalid # value and test that it does not match 'schema_type'. - for schema_name, (schema_type, valid_schema) in six.iteritems(valid_schemas): + for schema_name, (schema_type, valid_schema) in valid_schemas.items(): invalid_schema = 0xBAD if isinstance(schema_type, securesystemslib.schema.Integer): invalid_schema = 'BAD' @@ -779,7 +778,7 @@ def test_format_base64(self): # Test conditions for valid arguments. data = 'updateframework'.encode('utf-8') self.assertEqual('dXBkYXRlZnJhbWV3b3Jr', tuf.formats.format_base64(data)) - self.assertTrue(isinstance(tuf.formats.format_base64(data), six.string_types)) + self.assertTrue(isinstance(tuf.formats.format_base64(data), str)) # Test conditions for invalid arguments. self.assertRaises(securesystemslib.exceptions.FormatError, tuf.formats.format_base64, 123) @@ -791,7 +790,7 @@ def test_parse_base64(self): # Test conditions for valid arguments. base64 = 'dXBkYXRlZnJhbWV3b3Jr' self.assertEqual(b'updateframework', tuf.formats.parse_base64(base64)) - self.assertTrue(isinstance(tuf.formats.parse_base64(base64), six.binary_type)) + self.assertTrue(isinstance(tuf.formats.parse_base64(base64), bytes)) # Test conditions for invalid arguments. self.assertRaises(securesystemslib.exceptions.FormatError, tuf.formats.parse_base64, 123) diff --git a/tests/test_indefinite_freeze_attack.py b/tests/test_indefinite_freeze_attack.py index 6e10f3a075..1c3127e03f 100755 --- a/tests/test_indefinite_freeze_attack.py +++ b/tests/test_indefinite_freeze_attack.py @@ -53,6 +53,7 @@ import logging import unittest import sys +from urllib import request if sys.version_info >= (3, 3): import unittest.mock as mock @@ -71,7 +72,6 @@ from tests import utils import securesystemslib -import six # The repository tool is imported and logs console messages by default. Disable # console log messages generated by this unit test. @@ -223,7 +223,7 @@ def test_without_tuf(self): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'metadata', 'timestamp.json') - six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_timestamp_path) + request.urlretrieve(url_file.replace('\\', '/'), client_timestamp_path) length, hashes = securesystemslib.util.get_file_details(client_timestamp_path) download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes) @@ -367,7 +367,7 @@ def test_with_tuf(self): except tuf.exceptions.NoWorkingMirrorError as e: # Make sure the contained error is ExpiredMetadataError - for mirror_url, mirror_error in six.iteritems(e.mirror_errors): + for mirror_url, mirror_error in e.mirror_errors.items(): self.assertTrue(isinstance(mirror_error, tuf.exceptions.ExpiredMetadataError)) else: @@ -427,7 +427,7 @@ def test_with_tuf(self): except tuf.exceptions.NoWorkingMirrorError as e: # Make sure the contained error is ExpiredMetadataError - for mirror_url, mirror_error in six.iteritems(e.mirror_errors): + for mirror_url, mirror_error in e.mirror_errors.items(): self.assertTrue(isinstance(mirror_error, tuf.exceptions.ExpiredMetadataError)) self.assertTrue(mirror_url.endswith('snapshot.json')) diff --git a/tests/test_key_revocation_integration.py b/tests/test_key_revocation_integration.py index b235760e4a..0601e8ae75 100755 --- a/tests/test_key_revocation_integration.py +++ b/tests/test_key_revocation_integration.py @@ -55,7 +55,6 @@ from tests import utils import securesystemslib -import six logger = logging.getLogger(__name__) repo_tool.disable_console_log_messages() diff --git a/tests/test_log.py b/tests/test_log.py index cba1762f10..82637f50f5 100755 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -25,6 +25,7 @@ import os import shutil import sys +import importlib import tuf import tuf.log @@ -35,7 +36,6 @@ from tests import utils -from six.moves import reload_module # We explicitly create a logger which is a child of the tuf hierarchy, # instead of using the standard getLogger(__name__) pattern, because the @@ -96,7 +96,7 @@ def test_set_filehandler_log_level(self): # Test that the log level of the file handler cannot be set because # file logging is disabled (via tuf.settings.ENABLE_FILE_LOGGING). tuf.settings.ENABLE_FILE_LOGGING = False - reload_module(tuf.log) + importlib.reload(tuf.log) # Test for improperly formatted argument. self.assertRaises(securesystemslib.exceptions.FormatError, tuf.log.set_filehandler_log_level, '123') diff --git a/tests/test_mix_and_match_attack.py b/tests/test_mix_and_match_attack.py index 123c447325..1f509500e8 100755 --- a/tests/test_mix_and_match_attack.py +++ b/tests/test_mix_and_match_attack.py @@ -55,7 +55,6 @@ from tests import utils -import six # The repository tool is imported and logs console messages by default. # Disable console log messages generated by this unit test. @@ -227,7 +226,7 @@ def test_with_tuf(self): # 'tuf.exceptions.BadVersionNumberError' exception is raised by # each mirror. except tuf.exceptions.NoWorkingMirrorError as exception: - for mirror_url, mirror_error in six.iteritems(exception.mirror_errors): + for mirror_url, mirror_error in exception.mirror_errors.items(): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'metadata', 'role1.json') diff --git a/tests/test_multiple_repositories_integration.py b/tests/test_multiple_repositories_integration.py index de9920244d..63d8f63ac6 100755 --- a/tests/test_multiple_repositories_integration.py +++ b/tests/test_multiple_repositories_integration.py @@ -47,7 +47,6 @@ from tests import utils -import six import securesystemslib logger = logging.getLogger(__name__) @@ -271,7 +270,7 @@ def test_repository_tool(self): multi_repo_updater = updater.MultiRepoUpdater(self.map_file) valid_targetinfo = multi_repo_updater.get_valid_targetinfo('file3.txt') - for my_updater, my_targetinfo in six.iteritems(valid_targetinfo): + for my_updater, my_targetinfo in valid_targetinfo.items(): my_updater.download_target(my_targetinfo, self.temporary_directory) self.assertTrue(os.path.exists(os.path.join(self.temporary_directory, 'file3.txt'))) diff --git a/tests/test_replay_attack.py b/tests/test_replay_attack.py index 1195d99b22..cf26731466 100755 --- a/tests/test_replay_attack.py +++ b/tests/test_replay_attack.py @@ -45,6 +45,7 @@ import logging import unittest import sys +from urllib import request import tuf.formats import tuf.log @@ -55,7 +56,7 @@ from tests import utils import securesystemslib -import six + # The repository tool is imported and logs console messages by default. # Disable console log messages generated by this unit test. @@ -220,7 +221,7 @@ def test_without_tuf(self): self.repository_name, 'metadata', 'current', 'timestamp.json') # On Windows, the URL portion should not contain back slashes. - six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_timestamp_path) + request.urlretrieve(url_file.replace('\\', '/'), client_timestamp_path) length, hashes = securesystemslib.util.get_file_details(client_timestamp_path) download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes) @@ -233,7 +234,7 @@ def test_without_tuf(self): shutil.move(backup_timestamp, timestamp_path) # On Windows, the URL portion should not contain back slashes. - six.moves.urllib.request.urlretrieve(url_file.replace('\\', '/'), client_timestamp_path) + request.urlretrieve(url_file.replace('\\', '/'), client_timestamp_path) length, hashes = securesystemslib.util.get_file_details(client_timestamp_path) download_fileinfo = tuf.formats.make_targets_fileinfo(length, hashes) @@ -312,7 +313,7 @@ def test_with_tuf(self): # Verify that the specific 'tuf.exceptions.ReplayedMetadataError' is raised by each # mirror. except tuf.exceptions.NoWorkingMirrorError as exception: - for mirror_url, mirror_error in six.iteritems(exception.mirror_errors): + for mirror_url, mirror_error in exception.mirror_errors.items(): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'metadata', 'timestamp.json') diff --git a/tests/test_repository_lib.py b/tests/test_repository_lib.py index 36d1826a2e..ea31b1a443 100755 --- a/tests/test_repository_lib.py +++ b/tests/test_repository_lib.py @@ -57,7 +57,6 @@ import securesystemslib.rsa_keys import securesystemslib.interface import securesystemslib.storage -import six logger = logging.getLogger(__name__) @@ -271,7 +270,7 @@ def test_get_target_hash(self): '/README.txt': '8faee106f1bb69f34aaf1df1e3c2e87d763c4d878cb96b91db13495e32ceb0b0', '/packages/file2.txt': 'c9c4a5cdd84858dd6a23d98d7e6e6b2aec45034946c16b2200bc317c75415e92' } - for filepath, target_hash in six.iteritems(expected_target_hashes): + for filepath, target_hash in expected_target_hashes.items(): self.assertTrue(tuf.formats.RELPATH_SCHEMA.matches(filepath)) self.assertTrue(securesystemslib.formats.HASH_SCHEMA.matches(target_hash)) self.assertEqual(repo_lib.get_target_hash(filepath), target_hash) diff --git a/tests/test_slow_retrieval_attack.py b/tests/test_slow_retrieval_attack.py index 8a56e483ba..670d069b3f 100755 --- a/tests/test_slow_retrieval_attack.py +++ b/tests/test_slow_retrieval_attack.py @@ -60,8 +60,6 @@ from tests import utils -import six - logger = logging.getLogger(__name__) repo_tool.disable_console_log_messages() @@ -218,7 +216,7 @@ def test_delay_before_send(self): # Verify that the specific 'tuf.exceptions.SlowRetrievalError' exception is raised by # each mirror. except tuf.exceptions.NoWorkingMirrorError as exception: - for mirror_url, mirror_error in six.iteritems(exception.mirror_errors): + for mirror_url, mirror_error in exception.mirror_errors.items(): url_prefix = self.repository_mirrors['mirror1']['url_prefix'] url_file = os.path.join(url_prefix, 'targets', 'file1.txt') diff --git a/tests/test_updater.py b/tests/test_updater.py index 4654cfab23..e9fe2d6f2d 100755 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -79,7 +79,6 @@ from tests import utils import securesystemslib -import six logger = logging.getLogger(__name__) repo_tool.disable_console_log_messages() @@ -775,7 +774,7 @@ def test_3__update_metadata(self): DEFAULT_TARGETS_FILELENGTH, 88) except tuf.exceptions.NoWorkingMirrorError as e: - for mirror_error in six.itervalues(e.mirror_errors): + for mirror_error in e.mirror_errors.values(): assert isinstance(mirror_error, tuf.exceptions.BadVersionNumberError) else: @@ -791,7 +790,7 @@ def test_3__update_metadata(self): 88) except tuf.exceptions.NoWorkingMirrorError as e: - for mirror_error in six.itervalues(e.mirror_errors): + for mirror_error in e.mirror_errors.values(): assert isinstance(mirror_error, tuf.exceptions.BadVersionNumberError) else: @@ -839,7 +838,7 @@ def test_3__get_metadata_file(self): # Note that this test provides a piece of metadata which would fail to # be accepted -- with a different error -- if the specification version # number were not a problem. - for mirror_error in six.itervalues(e.mirror_errors): + for mirror_error in e.mirror_errors.values(): assert isinstance( mirror_error, tuf.exceptions.UnsupportedSpecificationError) @@ -921,7 +920,7 @@ def test_3__targets_of_role(self): # target files. self.assertTrue(tuf.formats.TARGETINFOS_SCHEMA.matches(targetinfos_list)) for targetinfo in targetinfos_list: - self.assertTrue((targetinfo['filepath'], targetinfo['fileinfo']) in six.iteritems(targets_in_metadata)) + self.assertTrue((targetinfo['filepath'], targetinfo['fileinfo']) in targets_in_metadata.items()) @@ -1084,7 +1083,7 @@ def test_5_targets_of_role(self): # target files. self.assertTrue(tuf.formats.TARGETINFOS_SCHEMA.matches(targetinfos)) for targetinfo in targetinfos: - self.assertTrue((targetinfo['filepath'], targetinfo['fileinfo']) in six.iteritems(expected_targets)) + self.assertTrue((targetinfo['filepath'], targetinfo['fileinfo']) in expected_targets.items()) # Test: Invalid arguments. # targets_of_role() expected a string rolename. @@ -1368,7 +1367,7 @@ def test_6_download_target(self): # field contains at least one confined target and excludes needed target # file. mirrors = self.repository_updater.mirrors - for mirror_name, mirror_info in six.iteritems(mirrors): + for mirror_name, mirror_info in mirrors.items(): mirrors[mirror_name]['confined_target_dirs'] = [self.random_path()] try: @@ -1630,7 +1629,7 @@ def test_9__get_target_hash(self): '/file1.txt': 'e3a3d89eb3b70ce3fbce6017d7b8c12d4abd5635427a0e8a238f53157df85b3d', '/Jalape\xc3\xb1o': '78bfd5c314680545eb48ecad508aceb861f8d6e680f4fe1b791da45c298cda88' } - for filepath, target_hash in six.iteritems(expected_target_hashes): + for filepath, target_hash in expected_target_hashes.items(): self.assertTrue(tuf.formats.RELPATH_SCHEMA.matches(filepath)) self.assertTrue(securesystemslib.formats.HASH_SCHEMA.matches(target_hash)) self.assertEqual(self.repository_updater._get_target_hash(filepath), target_hash) diff --git a/tests/test_updater_root_rotation_integration.py b/tests/test_updater_root_rotation_integration.py index 9182aa6c1f..d3a9879240 100755 --- a/tests/test_updater_root_rotation_integration.py +++ b/tests/test_updater_root_rotation_integration.py @@ -62,7 +62,6 @@ from tests import utils import securesystemslib -import six logger = logging.getLogger(__name__) repo_tool.disable_console_log_messages() @@ -248,7 +247,7 @@ def test_verify_root_with_current_keyids_and_threshold(self): with self.assertRaises(tuf.exceptions.NoWorkingMirrorError) as cm: self.repository_updater.refresh() - for mirror_url, mirror_error in six.iteritems(cm.exception.mirror_errors): + for mirror_url, mirror_error in cm.exception.mirror_errors.items(): self.assertTrue(mirror_url.endswith('/2.root.json')) self.assertTrue(isinstance(mirror_error, securesystemslib.exceptions.BadSignatureError)) @@ -307,7 +306,7 @@ def test_verify_root_with_duplicate_current_keyids(self): with self.assertRaises(tuf.exceptions.NoWorkingMirrorError) as cm: self.repository_updater.refresh() - for mirror_url, mirror_error in six.iteritems(cm.exception.mirror_errors): + for mirror_url, mirror_error in cm.exception.mirror_errors.items(): self.assertTrue(mirror_url.endswith('/2.root.json')) self.assertTrue(isinstance(mirror_error, securesystemslib.exceptions.BadSignatureError)) @@ -450,7 +449,7 @@ def test_root_rotation_missing_keys(self): with self.assertRaises(tuf.exceptions.NoWorkingMirrorError) as cm: self.repository_updater.refresh() - for mirror_url, mirror_error in six.iteritems(cm.exception.mirror_errors): + for mirror_url, mirror_error in cm.exception.mirror_errors.items(): self.assertTrue(mirror_url.endswith('/2.root.json')) self.assertTrue(isinstance(mirror_error, securesystemslib.exceptions.BadSignatureError)) @@ -525,7 +524,7 @@ def test_root_rotation_unmet_last_version_threshold(self): with self.assertRaises(tuf.exceptions.NoWorkingMirrorError) as cm: self.repository_updater.refresh() - for mirror_url, mirror_error in six.iteritems(cm.exception.mirror_errors): + for mirror_url, mirror_error in cm.exception.mirror_errors.items(): self.assertTrue(mirror_url.endswith('/3.root.json')) self.assertTrue(isinstance(mirror_error, securesystemslib.exceptions.BadSignatureError)) @@ -571,7 +570,7 @@ def test_root_rotation_unmet_new_threshold(self): with self.assertRaises(tuf.exceptions.NoWorkingMirrorError) as cm: self.repository_updater.refresh() - for mirror_url, mirror_error in six.iteritems(cm.exception.mirror_errors): + for mirror_url, mirror_error in cm.exception.mirror_errors.items(): self.assertTrue(mirror_url.endswith('/3.root.json')) self.assertTrue(isinstance(mirror_error, securesystemslib.exceptions.BadSignatureError)) @@ -606,7 +605,7 @@ def test_root_rotation_discard_untrusted_version(self): with self.assertRaises(tuf.exceptions.NoWorkingMirrorError) as cm: self.repository_updater.refresh() - for mirror_url, mirror_error in six.iteritems(cm.exception.mirror_errors): + for mirror_url, mirror_error in cm.exception.mirror_errors.items(): self.assertTrue(mirror_url.endswith('/2.root.json')) self.assertTrue(isinstance(mirror_error, securesystemslib.exceptions.BadSignatureError)) diff --git a/tuf/client/updater.py b/tuf/client/updater.py index 8084dffa64..b353d631ef 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -128,7 +128,6 @@ import time import fnmatch import copy -import six import warnings import io @@ -388,7 +387,7 @@ def _matching_targetinfo( # a threshold of 2: # [A, B, C, B, A, C] # In this case, targetinfo B is returned. - for valid_updater, compared_targetinfo in six.iteritems(valid_targetinfo): + for valid_updater, compared_targetinfo in valid_targetinfo.items(): if not self._targetinfo_match( targetinfo, compared_targetinfo, match_custom_field): @@ -960,7 +959,7 @@ def _import_delegations(self, parent_role): logger.debug('Adding roles delegated from ' + repr(parent_role) + '.') # Iterate the keys of the delegated roles of 'parent_role' and load them. - for keyid, keyinfo in six.iteritems(keys_info): + for keyid, keyinfo in keys_info.items(): if keyinfo['keytype'] in ['rsa', 'ed25519', 'ecdsa-sha2-nistp256']: # We specify the keyid to ensure that it's the correct keyid @@ -1205,7 +1204,7 @@ def _check_hashes(self, file_object, trusted_hashes): """ # Verify each hash, raise an exception if any hash fails to verify - for algorithm, trusted_hash in six.iteritems(trusted_hashes): + for algorithm, trusted_hash in trusted_hashes.items(): digest_object = sslib_hash.digest_fileobject(file_object, algorithm) computed_hash = digest_object.hexdigest() @@ -1565,9 +1564,8 @@ def _get_metadata_file(self, metadata_role, remote_filename, ". The update will continue as the major versions match.") except (ValueError, TypeError) as error: - six.raise_from(sslib_exceptions.FormatError('Improperly' - ' formatted spec_version, which must be in major.minor.fix format'), - error) + raise sslib_exceptions.FormatError('Improperly' + ' formatted spec_version, which must be in major.minor.fix format') from error # If the version number is unspecified, ensure that the version number # downloaded is greater than the currently trusted version number for @@ -2099,7 +2097,7 @@ def _fileinfo_has_changed(self, metadata_filename, new_fileinfo): # without having that result in considering all files as needing to be # updated, or not all hash algorithms listed can be calculated on the # specific client. - for algorithm, hash_value in six.iteritems(new_fileinfo['hashes']): + for algorithm, hash_value in new_fileinfo['hashes'].items(): # We're only looking for a single match. This isn't a security # check, we just want to prevent unnecessary downloads. if algorithm in current_fileinfo['hashes']: @@ -2403,7 +2401,7 @@ def _refresh_targets_metadata(self, rolename='targets', if refresh_all_delegated_roles: - for role in six.iterkeys(self.metadata['current']['snapshot']['meta']): + for role in self.metadata['current']['snapshot']['meta'].keys(): # snapshot.json keeps track of root.json, targets.json, and delegated # roles (e.g., django.json, unclaimed.json). Remove the 'targets' role # because it gets updated when the targets.json file is updated in @@ -2493,7 +2491,7 @@ def _targets_of_role(self, rolename, targets=None, skip_refresh=False): return [] # Get the targets specified by the role itself. - for filepath, fileinfo in six.iteritems(self.metadata['current'][rolename].get('targets', [])): + for filepath, fileinfo in self.metadata['current'][rolename].get('targets', []).items(): new_target = {} new_target['filepath'] = filepath new_target['fileinfo'] = fileinfo @@ -3084,7 +3082,7 @@ def updated_targets(self, targets, destination_directory): # Try one of the algorithm/digest combos for a mismatch. We break # as soon as we find a mismatch. - for algorithm, digest in six.iteritems(target['fileinfo']['hashes']): + for algorithm, digest in target['fileinfo']['hashes'].items(): digest_object = None try: digest_object = sslib_hash.digest_filename(target_filepath, diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index df1c17c212..635b92ea87 100755 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -37,7 +37,6 @@ import shutil import tempfile import json -import six import securesystemslib # pylint: disable=unused-import @@ -948,7 +947,7 @@ def load_project(project_directory, prefix='', new_targets_location=None, roleinfo['expires'] = metadata_object['expires'] roleinfo['paths'] = {} - for filepath, fileinfo in six.iteritems(metadata_object['targets']): + for filepath, fileinfo in metadata_object['targets'].items(): roleinfo['paths'].update({filepath: fileinfo.get('custom', {})}) roleinfo['delegations'] = metadata_object['delegations'] roleinfo['partial_loaded'] = False diff --git a/tuf/download.py b/tuf/download.py index 6b56ba2569..af013c71eb 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -32,9 +32,9 @@ from __future__ import unicode_literals import logging -import six import timeit import tempfile +from urllib import parse import securesystemslib # pylint: disable=unused-import from securesystemslib import formats as sslib_formats @@ -185,7 +185,7 @@ def _download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True): # This converts it to the common format. unquote() replaces %xx escapes in a # url with their single-character equivalent. A back-slash may be encoded as # %5c in the url, which should also be replaced with a forward slash. - url = six.moves.urllib.parse.unquote(url).replace('\\', '/') + url = parse.unquote(url).replace('\\', '/') logger.info('Downloading: ' + repr(url)) # This is the temporary file that we will return to contain the contents of diff --git a/tuf/exceptions.py b/tuf/exceptions.py index 5b2a345c9c..804f6f2dd8 100755 --- a/tuf/exceptions.py +++ b/tuf/exceptions.py @@ -30,7 +30,7 @@ from __future__ import division from __future__ import unicode_literals -import six +from urllib import parse import logging logger = logging.getLogger(__name__) @@ -296,10 +296,10 @@ def __init__(self, mirror_errors): def __str__(self): all_errors = 'No working mirror was found:' - for mirror_url, mirror_error in six.iteritems(self.mirror_errors): + for mirror_url, mirror_error in self.mirror_errors.items(): try: # http://docs.python.org/2/library/urlparse.html#urlparse.urlparse - mirror_url_tokens = six.moves.urllib.parse.urlparse(mirror_url) + mirror_url_tokens = parse.urlparse(mirror_url) except Exception: logger.exception('Failed to parse mirror URL: ' + repr(mirror_url)) diff --git a/tuf/formats.py b/tuf/formats.py index 9527b51223..e0bc179edc 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -76,8 +76,6 @@ import tuf from tuf import exceptions -import six - # As per TUF spec 1.0.0 the spec version field must follow the Semantic # Versioning 2.0.0 (semver) format. The regex pattern is provided by semver. # https://semver.org/spec/v2.0.0.html#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string @@ -638,9 +636,8 @@ def expiry_string_to_datetime(expires): try: return datetime.datetime.strptime(expires, "%Y-%m-%dT%H:%M:%SZ") except ValueError as error: - six.raise_from(sslib_exceptions.FormatError( - 'Failed to parse ' + repr(expires) + ' as an expiry time'), - error) + raise sslib_exceptions.FormatError( + 'Failed to parse ' + repr(expires) + ' as an expiry time') from error @@ -781,7 +778,7 @@ def parse_base64(base64_string): 'base64_string'. """ - if not isinstance(base64_string, six.string_types): + if not isinstance(base64_string, str): message = 'Invalid argument: '+repr(base64_string) raise sslib_exceptions.FormatError(message) @@ -991,15 +988,14 @@ def check_signable_object_format(signable): role_type = signable['signed']['_type'] except (KeyError, TypeError) as error: - six.raise_from(sslib_exceptions.FormatError( - 'Untyped signable object.'), error) + raise sslib_exceptions.FormatError('Untyped signable object.') from error try: schema = SCHEMAS_BY_TYPE[role_type] except KeyError as error: - six.raise_from(sslib_exceptions.FormatError( - 'Unrecognized type ' + repr(role_type)), error) + raise sslib_exceptions.FormatError('Unrecognized type ' + + repr(role_type)) from error if not signable['signatures']: raise exceptions.UnsignedMetadataError('Signable object of type ' + diff --git a/tuf/keydb.py b/tuf/keydb.py index 71ef1058a4..9637bb6692 100755 --- a/tuf/keydb.py +++ b/tuf/keydb.py @@ -52,8 +52,6 @@ from tuf import exceptions from tuf import formats -import six - # List of strings representing the key types supported by TUF. _SUPPORTED_KEY_TYPES = ['rsa', 'ed25519', 'ecdsa-sha2-nistp256'] @@ -119,7 +117,7 @@ def create_keydb_from_root_metadata(root_metadata, repository_name='default'): # Iterate the keys found in 'root_metadata' by converting them to # 'RSAKEY_SCHEMA' if their type is 'rsa', and then adding them to the # key database using the provided keyid. - for keyid, key_metadata in six.iteritems(root_metadata['keys']): + for keyid, key_metadata in root_metadata['keys'].items(): if key_metadata['keytype'] in _SUPPORTED_KEY_TYPES: # 'key_metadata' is stored in 'KEY_SCHEMA' format. Call # create_from_metadata_format() to get the key in 'RSAKEY_SCHEMA' format, @@ -348,7 +346,7 @@ def get_key(keyid, repository_name='default'): return copy.deepcopy(_keydb_dict[repository_name][keyid]) except KeyError as error: - six.raise_from(exceptions.UnknownKeyError('Key: ' + keyid), error) + raise exceptions.UnknownKeyError('Key: ' + keyid) from error diff --git a/tuf/mirrors.py b/tuf/mirrors.py index 78d5053402..8a1281591a 100755 --- a/tuf/mirrors.py +++ b/tuf/mirrors.py @@ -31,6 +31,7 @@ from __future__ import unicode_literals import os +from urllib import parse import securesystemslib # pylint: disable=unused-import from securesystemslib import exceptions as sslib_exceptions @@ -39,7 +40,6 @@ from tuf import formats -import six # The type of file to be downloaded from a repository. The # 'get_list_of_mirrors' function supports these file types. @@ -98,7 +98,7 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict): path_key = 'metadata_path' if file_type == 'meta' else 'targets_path' list_of_mirrors = [] - for junk, mirror_info in six.iteritems(mirrors_dict): + for junk, mirror_info in mirrors_dict.items(): # Does mirror serve this file type at all? path = mirror_info.get(path_key) if path is None: @@ -114,12 +114,12 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict): confined_target_dirs): continue - # urllib.quote(string) replaces special characters in string using the %xx + # parse.quote(string) replaces special characters in string using the %xx # escape. This is done to avoid parsing issues of the URL on the server # side. Do *NOT* pass URLs with Unicode characters without first encoding # the URL as UTF-8. We need a long-term solution with #61. # http://bugs.python.org/issue1712522 - file_path = six.moves.urllib.parse.quote(file_path) + file_path = parse.quote(file_path) url = os.path.join(mirror_info['url_prefix'], path, file_path) # The above os.path.join() result as well as input file_path may be diff --git a/tuf/repository_lib.py b/tuf/repository_lib.py index d3158cb6c9..c7bbde4ba3 100644 --- a/tuf/repository_lib.py +++ b/tuf/repository_lib.py @@ -37,7 +37,6 @@ import logging import shutil import json -import six import tempfile import securesystemslib # pylint: disable=unused-import @@ -542,8 +541,8 @@ def _load_top_level_metadata(repository, top_level_filenames, repository_name): consistent_snapshot = root_metadata['consistent_snapshot'] except sslib_exceptions.StorageError as error: - six.raise_from(exceptions.RepositoryError('Cannot load the required' - ' root file: ' + repr(root_filename)), error) + raise exceptions.RepositoryError('Cannot load the required' + ' root file: ' + repr(root_filename)) from error # Load 'timestamp.json'. A Timestamp role file without a version number is # always written. @@ -571,8 +570,8 @@ def _load_top_level_metadata(repository, top_level_filenames, repository_name): repository_name=repository_name) except sslib_exceptions.StorageError as error: - six.raise_from(exceptions.RepositoryError('Cannot load the Timestamp ' - 'file: ' + repr(timestamp_filename)), error) + raise exceptions.RepositoryError('Cannot load the Timestamp ' + 'file: ' + repr(timestamp_filename)) from error # Load 'snapshot.json'. A consistent snapshot.json must be calculated if # 'consistent_snapshot' is True. @@ -617,8 +616,8 @@ def _load_top_level_metadata(repository, top_level_filenames, repository_name): repository_name=repository_name) except sslib_exceptions.StorageError as error: - six.raise_from(exceptions.RepositoryError('The Snapshot file ' - 'cannot be loaded: '+ repr(snapshot_filename)), error) + raise exceptions.RepositoryError('The Snapshot file ' + 'cannot be loaded: '+ repr(snapshot_filename)) from error # Load 'targets.json'. A consistent snapshot of the Targets role must be # calculated if 'consistent_snapshot' is True. @@ -661,7 +660,7 @@ def _load_top_level_metadata(repository, top_level_filenames, repository_name): repository_name=repository_name) # Add the keys specified in the delegations field of the Targets role. - for keyid, key_metadata in six.iteritems(targets_metadata['delegations']['keys']): + for keyid, key_metadata in targets_metadata['delegations']['keys'].items(): # Use the keyid found in the delegation key_object, _ = sslib_keys.format_metadata_to_key(key_metadata, @@ -680,8 +679,8 @@ def _load_top_level_metadata(repository, top_level_filenames, repository_name): pass except sslib_exceptions.StorageError as error: - six.raise_from(exceptions.RepositoryError('The Targets file ' - 'can not be loaded: ' + repr(targets_filename)), error) + raise exceptions.RepositoryError('The Targets file ' + 'can not be loaded: ' + repr(targets_filename)) from error return repository, consistent_snapshot @@ -1428,7 +1427,7 @@ def generate_targets_metadata(targets_directory, target_files, version, if use_existing_fileinfo: # Use the provided fileinfo dicts, conforming to FILEINFO_SCHEMA, rather than # generating fileinfo - for target, fileinfo in six.iteritems(target_files): + for target, fileinfo in target_files.items(): # Ensure all fileinfo entries in target_files have a non-empty hashes dict if not fileinfo.get('hashes', None): @@ -1497,7 +1496,7 @@ def _generate_targets_fileinfo(target_files, targets_directory, filedict = {} # Generate the fileinfo of all the target files listed in 'target_files'. - for target, fileinfo in six.iteritems(target_files): + for target, fileinfo in target_files.items(): # The root-most folder of the targets directory should not be included in # target paths listed in targets metadata. @@ -1518,7 +1517,7 @@ def _generate_targets_fileinfo(target_files, targets_directory, # Copy 'target_path' to 'digest_target' if consistent hashing is enabled. if write_consistent_targets: - for target_digest in six.itervalues(filedict[relative_targetpath]['hashes']): + for target_digest in filedict[relative_targetpath]['hashes'].values(): dirname, basename = os.path.split(target_path) digest_filename = target_digest + '.' + basename digest_target = os.path.join(dirname, digest_filename) diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index 9d195da000..7154274b94 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -39,7 +39,6 @@ import tempfile import shutil import json -import six from collections import deque @@ -3160,7 +3159,7 @@ def load_repository(repository_directory, repository_name='default', # log a warning here as there may be many such duplicate key warnings. # The repository maintainer should have also been made aware of the # duplicate key when it was added. - for key_metadata in six.itervalues(metadata_object['delegations']['keys']): + for key_metadata in metadata_object['delegations']['keys'].values(): # The repo may have used hashing algorithms for the generated keyids # that doesn't match the client's set of hash algorithms. Make sure diff --git a/tuf/requests_fetcher.py b/tuf/requests_fetcher.py index 25a2f9d0db..1692ebee7c 100644 --- a/tuf/requests_fetcher.py +++ b/tuf/requests_fetcher.py @@ -7,9 +7,9 @@ # Imports import requests -import six import logging import time +from urllib import parse from urllib3.exceptions import ReadTimeoutError import tuf @@ -137,7 +137,7 @@ def _get_session(self, url): """ # Use a different requests.Session per schema+hostname combination, to # reuse connections while minimizing subtle security issues. - parsed_url = six.moves.urllib.parse.urlparse(url) + parsed_url = parse.urlparse(url) if not parsed_url.scheme or not parsed_url.hostname: raise exceptions.URLParsingError( diff --git a/tuf/roledb.py b/tuf/roledb.py index 62f83bb8ae..26a1160723 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -59,8 +59,6 @@ from tuf import exceptions from tuf import formats -import six - # See 'tuf.log' to learn how logging is handled in TUF. logger = logging.getLogger(__name__) @@ -135,7 +133,7 @@ def create_roledb_from_root_metadata(root_metadata, repository_name='default'): # Iterate the roles found in 'root_metadata' and add them to '_roledb_dict'. # Duplicates are avoided. - for rolename, roleinfo in six.iteritems(root_metadata['roles']): + for rolename, roleinfo in root_metadata['roles'].items(): if rolename == 'root': roleinfo['version'] = root_metadata['version'] roleinfo['expires'] = root_metadata['expires'] diff --git a/tuf/scripts/repo.py b/tuf/scripts/repo.py index 5866a8853c..d9af027d72 100755 --- a/tuf/scripts/repo.py +++ b/tuf/scripts/repo.py @@ -146,7 +146,6 @@ import shutil import time import fnmatch -import six import securesystemslib # pylint: disable=unused-import from securesystemslib import exceptions as sslib_exceptions @@ -469,9 +468,9 @@ def import_privatekey_from_file(keypath, password=None): encrypted_key, 'rsassa-pss-sha256', password) except sslib_exceptions.CryptoError as error: - six.raise_from(exceptions.Error(repr(keypath) + ' cannot be ' + raise exceptions.Error(repr(keypath) + ' cannot be ' ' imported, possibly because an invalid key file is given or ' - ' the decryption password is incorrect.'), error) + ' the decryption password is incorrect.') from error if key_object['keytype'] not in SUPPORTED_KEY_TYPES: raise exceptions.Error('Trying to import an unsupported key' @@ -752,7 +751,7 @@ def remove_target_files_from_metadata(parsed_arguments, repository): parsed_arguments.role, repository._repository_name) for glob_pattern in parsed_arguments.remove: - for path in list(six.iterkeys(roleinfo['paths'])): + for path in list(roleinfo['paths'].keys()): if fnmatch.fnmatch(path, glob_pattern): del roleinfo['paths'][path] From ec68bd931696782c37ec5949adc871888d788a42 Mon Sep 17 00:00:00 2001 From: Kainaat Singh Date: Thu, 8 Apr 2021 16:45:27 +0200 Subject: [PATCH 09/17] Remove future module #1297 Signed-off-by: Kainaat Singh remove unwanted lines --- tests/aggregate_tests.py | 8 -------- tests/simple_https_server.py | 8 -------- tests/simple_server.py | 8 -------- tests/slow_retrieval_server.py | 8 -------- tests/test_arbitrary_package_attack.py | 8 -------- tests/test_download.py | 8 -------- tests/test_endless_data_attack.py | 8 -------- tests/test_extraneous_dependencies_attack.py | 8 -------- tests/test_fetcher.py | 3 --- tests/test_formats.py | 8 -------- tests/test_indefinite_freeze_attack.py | 8 -------- tests/test_key_revocation_integration.py | 8 -------- tests/test_keydb.py | 8 -------- tests/test_mirrors.py | 8 -------- tests/test_mix_and_match_attack.py | 8 -------- tests/test_multiple_repositories_integration.py | 8 -------- tests/test_replay_attack.py | 8 -------- tests/test_repository_lib.py | 8 -------- tests/test_repository_tool.py | 8 -------- tests/test_roledb.py | 8 -------- tests/test_root_versioning_integration.py | 4 ---- tests/test_sig.py | 8 -------- tests/test_slow_retrieval_attack.py | 8 -------- tests/test_tutorial.py | 7 ------- tests/test_unittest_toolbox.py | 8 -------- tests/test_updater.py | 8 -------- tests/test_updater_root_rotation_integration.py | 8 -------- tuf/client/updater.py | 8 -------- tuf/developer_tool.py | 7 ------- tuf/download.py | 8 -------- tuf/exceptions.py | 8 -------- tuf/formats.py | 8 -------- tuf/keydb.py | 8 -------- tuf/log.py | 8 -------- tuf/mirrors.py | 8 -------- tuf/repository_lib.py | 8 -------- tuf/repository_tool.py | 8 -------- tuf/roledb.py | 8 -------- tuf/scripts/client.py | 8 -------- tuf/scripts/repo.py | 8 -------- tuf/settings.py | 8 -------- tuf/sig.py | 8 -------- tuf/unittest_toolbox.py | 8 -------- 43 files changed, 333 deletions(-) diff --git a/tests/aggregate_tests.py b/tests/aggregate_tests.py index 49528e9034..ad87769e3c 100755 --- a/tests/aggregate_tests.py +++ b/tests/aggregate_tests.py @@ -26,14 +26,6 @@ 'tuf/tests'. Use --random to run the tests in random order. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import sys import unittest diff --git a/tests/simple_https_server.py b/tests/simple_https_server.py index ad54a4626b..29b6e40ab7 100755 --- a/tests/simple_https_server.py +++ b/tests/simple_https_server.py @@ -29,14 +29,6 @@ http://docs.python.org/library/simplehttpserver.html#module-SimpleHTTPServer """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import sys import ssl import os diff --git a/tests/simple_server.py b/tests/simple_server.py index cbdcced5d3..74e84f0d80 100755 --- a/tests/simple_server.py +++ b/tests/simple_server.py @@ -25,14 +25,6 @@ http://docs.python.org/library/simplehttpserver.html#module-SimpleHTTPServer """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import sys import random import socketserver diff --git a/tests/slow_retrieval_server.py b/tests/slow_retrieval_server.py index 44c764ba67..06ec495213 100755 --- a/tests/slow_retrieval_server.py +++ b/tests/slow_retrieval_server.py @@ -21,14 +21,6 @@ interval 'DELAY'). The server is used in 'test_slow_retrieval_attack.py'. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import sys import time diff --git a/tests/test_arbitrary_package_attack.py b/tests/test_arbitrary_package_attack.py index b31ffe3167..ab68d287a1 100755 --- a/tests/test_arbitrary_package_attack.py +++ b/tests/test_arbitrary_package_attack.py @@ -28,14 +28,6 @@ There is no difference between 'updates' and 'target' files. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import tempfile import shutil diff --git a/tests/test_download.py b/tests/test_download.py index 6ae0e4c4e4..81e2866407 100755 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -25,14 +25,6 @@ TODO: Adopt the environment variable management from test_proxy_use.py here. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import hashlib import logging import os diff --git a/tests/test_endless_data_attack.py b/tests/test_endless_data_attack.py index 4cc4e9370a..42bffd1f77 100755 --- a/tests/test_endless_data_attack.py +++ b/tests/test_endless_data_attack.py @@ -31,14 +31,6 @@ There is no difference between 'updates' and 'target' files. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import tempfile import shutil diff --git a/tests/test_extraneous_dependencies_attack.py b/tests/test_extraneous_dependencies_attack.py index a75ca904d4..e94a466072 100755 --- a/tests/test_extraneous_dependencies_attack.py +++ b/tests/test_extraneous_dependencies_attack.py @@ -34,14 +34,6 @@ There is no difference between 'updates' and 'target' files. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import tempfile import shutil diff --git a/tests/test_fetcher.py b/tests/test_fetcher.py index 312ef80959..0df0a4824c 100644 --- a/tests/test_fetcher.py +++ b/tests/test_fetcher.py @@ -5,9 +5,6 @@ """Unit test for RequestsFetcher. """ -# Help with Python 2 compatibility, where the '/' operator performs -# integer division. -from __future__ import division import logging import os diff --git a/tests/test_formats.py b/tests/test_formats.py index 5a657689fd..96da912b12 100755 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -20,14 +20,6 @@ Unit test for 'formats.py' """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import unittest import datetime import sys diff --git a/tests/test_indefinite_freeze_attack.py b/tests/test_indefinite_freeze_attack.py index 1c3127e03f..acf192704d 100755 --- a/tests/test_indefinite_freeze_attack.py +++ b/tests/test_indefinite_freeze_attack.py @@ -36,14 +36,6 @@ metadata without the client being aware. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import datetime import os import time diff --git a/tests/test_key_revocation_integration.py b/tests/test_key_revocation_integration.py index 0601e8ae75..bb720f462f 100755 --- a/tests/test_key_revocation_integration.py +++ b/tests/test_key_revocation_integration.py @@ -29,14 +29,6 @@ 'tests/unittest_toolbox.py'. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import shutil import tempfile diff --git a/tests/test_keydb.py b/tests/test_keydb.py index 1db0b421db..b075dd4acb 100755 --- a/tests/test_keydb.py +++ b/tests/test_keydb.py @@ -20,14 +20,6 @@ Unit test for 'keydb.py'. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import unittest import logging import sys diff --git a/tests/test_mirrors.py b/tests/test_mirrors.py index 5f49294876..ed87ff18e6 100755 --- a/tests/test_mirrors.py +++ b/tests/test_mirrors.py @@ -20,14 +20,6 @@ Unit test for 'mirrors.py'. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import unittest import sys diff --git a/tests/test_mix_and_match_attack.py b/tests/test_mix_and_match_attack.py index 1f509500e8..781a03e518 100755 --- a/tests/test_mix_and_match_attack.py +++ b/tests/test_mix_and_match_attack.py @@ -30,14 +30,6 @@ Note: There is no difference between 'updates' and 'target' files. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import tempfile import shutil diff --git a/tests/test_multiple_repositories_integration.py b/tests/test_multiple_repositories_integration.py index 63d8f63ac6..061105a49b 100755 --- a/tests/test_multiple_repositories_integration.py +++ b/tests/test_multiple_repositories_integration.py @@ -21,14 +21,6 @@ multiple repositories and separate sets of metadata for each. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import tempfile import logging diff --git a/tests/test_replay_attack.py b/tests/test_replay_attack.py index cf26731466..09bf732e87 100755 --- a/tests/test_replay_attack.py +++ b/tests/test_replay_attack.py @@ -30,14 +30,6 @@ Note: There is no difference between 'updates' and 'target' files. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import tempfile import datetime diff --git a/tests/test_repository_lib.py b/tests/test_repository_lib.py index ea31b1a443..96dcb0e0e0 100755 --- a/tests/test_repository_lib.py +++ b/tests/test_repository_lib.py @@ -20,14 +20,6 @@ Unit test for 'repository_lib.py'. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import time import datetime diff --git a/tests/test_repository_tool.py b/tests/test_repository_tool.py index 0fac025a31..be0333c351 100755 --- a/tests/test_repository_tool.py +++ b/tests/test_repository_tool.py @@ -20,14 +20,6 @@ Unit test for 'repository_tool.py'. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import time import datetime diff --git a/tests/test_roledb.py b/tests/test_roledb.py index 99f8d05368..73405b21bd 100755 --- a/tests/test_roledb.py +++ b/tests/test_roledb.py @@ -20,14 +20,6 @@ Unit test for 'roledb.py'. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import unittest import logging import sys diff --git a/tests/test_root_versioning_integration.py b/tests/test_root_versioning_integration.py index b0eda37bd5..5012029802 100755 --- a/tests/test_root_versioning_integration.py +++ b/tests/test_root_versioning_integration.py @@ -20,10 +20,6 @@ Test root versioning for efficient root key rotation. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals import os import logging diff --git a/tests/test_sig.py b/tests/test_sig.py index cb836bd7dd..a49c59c21c 100755 --- a/tests/test_sig.py +++ b/tests/test_sig.py @@ -21,14 +21,6 @@ Test cases for sig.py. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import unittest import logging import copy diff --git a/tests/test_slow_retrieval_attack.py b/tests/test_slow_retrieval_attack.py index 670d069b3f..6fdcb549f3 100755 --- a/tests/test_slow_retrieval_attack.py +++ b/tests/test_slow_retrieval_attack.py @@ -36,14 +36,6 @@ use slow target download. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import tempfile import shutil diff --git a/tests/test_tutorial.py b/tests/test_tutorial.py index 3fb3aaadfb..40f2fd2285 100755 --- a/tests/test_tutorial.py +++ b/tests/test_tutorial.py @@ -21,13 +21,6 @@ """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals import unittest import datetime # part of TUTORIAL.md diff --git a/tests/test_unittest_toolbox.py b/tests/test_unittest_toolbox.py index acc6f44371..5bd4169c87 100755 --- a/tests/test_unittest_toolbox.py +++ b/tests/test_unittest_toolbox.py @@ -20,14 +20,6 @@ Test cases for unittest_toolbox.py. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import unittest import logging import shutil diff --git a/tests/test_updater.py b/tests/test_updater.py index e9fe2d6f2d..3982f224be 100755 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -41,14 +41,6 @@ less dependent than 2. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import time import shutil diff --git a/tests/test_updater_root_rotation_integration.py b/tests/test_updater_root_rotation_integration.py index d3a9879240..51f85fd158 100755 --- a/tests/test_updater_root_rotation_integration.py +++ b/tests/test_updater_root_rotation_integration.py @@ -33,14 +33,6 @@ less dependent than 2. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import shutil import tempfile diff --git a/tuf/client/updater.py b/tuf/client/updater.py index b353d631ef..b67b19f07d 100755 --- a/tuf/client/updater.py +++ b/tuf/client/updater.py @@ -113,14 +113,6 @@ target_custom_data = target['fileinfo']['custom'] """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import errno import logging import os diff --git a/tuf/developer_tool.py b/tuf/developer_tool.py index 635b92ea87..e900651492 100755 --- a/tuf/developer_tool.py +++ b/tuf/developer_tool.py @@ -24,13 +24,6 @@ 'developer_tool.py'. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division - import os import errno import logging diff --git a/tuf/download.py b/tuf/download.py index af013c71eb..af12af614b 100755 --- a/tuf/download.py +++ b/tuf/download.py @@ -23,14 +23,6 @@ metadata of that file. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import logging import timeit import tempfile diff --git a/tuf/exceptions.py b/tuf/exceptions.py index 804f6f2dd8..baed5446eb 100755 --- a/tuf/exceptions.py +++ b/tuf/exceptions.py @@ -22,14 +22,6 @@ there is a good reason not to, and provide that reason in those cases. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - from urllib import parse import logging diff --git a/tuf/formats.py b/tuf/formats.py index e0bc179edc..ace8a99fb7 100755 --- a/tuf/formats.py +++ b/tuf/formats.py @@ -55,14 +55,6 @@ signable_object = make_signable(unsigned_object) """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import binascii import calendar import datetime diff --git a/tuf/keydb.py b/tuf/keydb.py index 9637bb6692..58733c75ea 100755 --- a/tuf/keydb.py +++ b/tuf/keydb.py @@ -33,14 +33,6 @@ dictionary's 'keyid' key (i.e., rsakey['keyid']). """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import logging import copy diff --git a/tuf/log.py b/tuf/log.py index 368845333e..485d2391df 100755 --- a/tuf/log.py +++ b/tuf/log.py @@ -61,14 +61,6 @@ http://docs.python.org/2/howto/logging-cookbook.html """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import logging import time diff --git a/tuf/mirrors.py b/tuf/mirrors.py index 8a1281591a..c7662d3eec 100755 --- a/tuf/mirrors.py +++ b/tuf/mirrors.py @@ -22,14 +22,6 @@ of the file with respect to the base url. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os from urllib import parse diff --git a/tuf/repository_lib.py b/tuf/repository_lib.py index c7bbde4ba3..bc15d0e238 100644 --- a/tuf/repository_lib.py +++ b/tuf/repository_lib.py @@ -23,14 +23,6 @@ complete guide to using 'tuf.repository_tool.py'. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import errno import time diff --git a/tuf/repository_tool.py b/tuf/repository_tool.py index 7154274b94..7ab22463e6 100755 --- a/tuf/repository_tool.py +++ b/tuf/repository_tool.py @@ -24,14 +24,6 @@ 'tuf.repository_tool.py'. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import time import datetime diff --git a/tuf/roledb.py b/tuf/roledb.py index 26a1160723..53d1c094f0 100755 --- a/tuf/roledb.py +++ b/tuf/roledb.py @@ -41,14 +41,6 @@ optional. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import logging import copy diff --git a/tuf/scripts/client.py b/tuf/scripts/client.py index c1c7bd7cb2..8f30c53648 100755 --- a/tuf/scripts/client.py +++ b/tuf/scripts/client.py @@ -59,14 +59,6 @@ $ client.py --repo http://localhost:8001 README.txt """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import sys import argparse import logging diff --git a/tuf/scripts/repo.py b/tuf/scripts/repo.py index d9af027d72..1dcaa73957 100755 --- a/tuf/scripts/repo.py +++ b/tuf/scripts/repo.py @@ -131,14 +131,6 @@ Delete repo in current working or specified directory. """ -# Help with Python 2+3 compatibility, where the print statement is a function, -# an implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import sys import logging diff --git a/tuf/settings.py b/tuf/settings.py index 2dcc8e3b25..f07c4d961a 100755 --- a/tuf/settings.py +++ b/tuf/settings.py @@ -23,14 +23,6 @@ behavior. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - # Set a directory that should be used for all temporary files. If this # is None, then the system default will be used. The system default diff --git a/tuf/sig.py b/tuf/sig.py index cc572ae647..4e1f05fc2a 100755 --- a/tuf/sig.py +++ b/tuf/sig.py @@ -40,14 +40,6 @@ is also a function for that. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import logging import securesystemslib # pylint: disable=unused-import diff --git a/tuf/unittest_toolbox.py b/tuf/unittest_toolbox.py index 95b64de315..a248380d2c 100755 --- a/tuf/unittest_toolbox.py +++ b/tuf/unittest_toolbox.py @@ -22,14 +22,6 @@ Specifically, Modified_TestCase is a derived class from unittest.TestCase. """ -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - import os import shutil import unittest From 1b7fca4910412ebcd5db6282ab1a85021e88fd81 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Mon, 12 Apr 2021 10:49:26 +0300 Subject: [PATCH 10/17] tests: Fix the order of rmdir and process kill Make sure test server processes are killed before the temporary directories are removed. Let Modified_Testcase handle the top-level temporary directory. Don't let Modified_testcase handle any subdirectories because: * teardown will try to remove them in the wrong order * removing the top level is enough Fixes #1344 Signed-off-by: Jussi Kukkonen --- .../test_multiple_repositories_integration.py | 15 +++++--------- tests/test_slow_retrieval_attack.py | 20 +++++-------------- tests/test_updater.py | 16 +++++---------- 3 files changed, 15 insertions(+), 36 deletions(-) diff --git a/tests/test_multiple_repositories_integration.py b/tests/test_multiple_repositories_integration.py index 061105a49b..0339a0d2d0 100755 --- a/tests/test_multiple_repositories_integration.py +++ b/tests/test_multiple_repositories_integration.py @@ -49,18 +49,16 @@ class TestMultipleRepositoriesIntegration(unittest_toolbox.Modified_TestCase): def setUp(self): - # We are inheriting from custom class. + # Modified_Testcase can handle temp dir removal unittest_toolbox.Modified_TestCase.setUp(self) - - self.temporary_directory = tempfile.mkdtemp(dir=os.getcwd()) + self.temporary_directory = self.make_temp_directory(directory=os.getcwd()) # Copy the original repository files provided in the test folder so that # any modifications made to repository files are restricted to the copies. # The 'repository_data' directory is expected to exist in 'tuf/tests/'. original_repository_files = os.path.join(os.getcwd(), 'repository_data') - self.temporary_repository_root = self.make_temp_directory(directory= - self.temporary_directory) + self.temporary_repository_root = tempfile.mkdtemp(dir=self.temporary_directory) # The original repository, keystore, and client directories will be copied # for each test case. @@ -151,10 +149,6 @@ def setUp(self): def tearDown(self): - # Modified_TestCase.tearDown() automatically deletes temporary files and - # directories that may have been created during each test case. - unittest_toolbox.Modified_TestCase.tearDown(self) - # Cleans the resources and flush the logged lines (if any). self.server_process_handler.clean() self.server_process_handler2.clean() @@ -163,7 +157,8 @@ def tearDown(self): tuf.roledb.clear_roledb(clear_all=True) tuf.keydb.clear_keydb(clear_all=True) - shutil.rmtree(self.temporary_directory) + # Remove top-level temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) def test_update(self): diff --git a/tests/test_slow_retrieval_attack.py b/tests/test_slow_retrieval_attack.py index 6fdcb549f3..6cf2e1a837 100755 --- a/tests/test_slow_retrieval_attack.py +++ b/tests/test_slow_retrieval_attack.py @@ -60,22 +60,17 @@ class TestSlowRetrieval(unittest_toolbox.Modified_TestCase): def setUp(self): - # We are inheriting from custom class. + # Modified_Testcase can handle temp dir removal unittest_toolbox.Modified_TestCase.setUp(self) + self.temporary_directory = self.make_temp_directory(directory=os.getcwd()) self.repository_name = 'test_repository1' - # Create a temporary directory to store the repository, metadata, and target - # files. 'temporary_directory' must be deleted in TearDownModule() so that - # temporary files are always removed, even when exceptions occur. - self.temporary_directory = tempfile.mkdtemp(dir=os.getcwd()) - # Copy the original repository files provided in the test folder so that # any modifications made to repository files are restricted to the copies. # The 'repository_data' directory is expected to exist in 'tuf/tests/'. original_repository_files = os.path.join(os.getcwd(), 'repository_data') - temporary_repository_root = \ - self.make_temp_directory(directory=self.temporary_directory) + temporary_repository_root = tempfile.mkdtemp(dir=self.temporary_directory) # The original repository, keystore, and client directories will be copied # for each test case. @@ -179,19 +174,14 @@ def setUp(self): def tearDown(self): - # Modified_TestCase.tearDown() automatically deletes temporary files and - # directories that may have been created during each test case. - unittest_toolbox.Modified_TestCase.tearDown(self) tuf.roledb.clear_roledb(clear_all=True) tuf.keydb.clear_keydb(clear_all=True) # Cleans the resources and flush the logged lines (if any). self.server_process_handler.clean() - # Remove the temporary repository directory, which should contain all the - # metadata, targets, and key files generated of all the test cases. - shutil.rmtree(self.temporary_directory) - + # Remove temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) def test_delay_before_send(self): diff --git a/tests/test_updater.py b/tests/test_updater.py index 3982f224be..7a527138dd 100755 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -1773,18 +1773,16 @@ def test_13__targets_of_role(self): class TestMultiRepoUpdater(unittest_toolbox.Modified_TestCase): def setUp(self): - # We are inheriting from custom class. + # Modified_Testcase can handle temp dir removal unittest_toolbox.Modified_TestCase.setUp(self) - - self.temporary_directory = tempfile.mkdtemp(dir=os.getcwd()) + self.temporary_directory = self.make_temp_directory(directory=os.getcwd()) # Copy the original repository files provided in the test folder so that # any modifications made to repository files are restricted to the copies. # The 'repository_data' directory is expected to exist in 'tuf/tests/'. original_repository_files = os.path.join(os.getcwd(), 'repository_data') - self.temporary_repository_root = self.make_temp_directory(directory= - self.temporary_directory) + self.temporary_repository_root = tempfile.mkdtemp(dir=self.temporary_directory) # Needed because in some tests simple_server.py cannot be found. # The reason is that the current working directory @@ -1899,9 +1897,6 @@ def setUp(self): def tearDown(self): - # Modified_TestCase.tearDown() automatically deletes temporary files and - # directories that may have been created during each test case. - unittest_toolbox.Modified_TestCase.tearDown(self) # Cleans the resources and flush the logged lines (if any). self.server_process_handler.clean() @@ -1911,9 +1906,8 @@ def tearDown(self): tuf.roledb.clear_roledb(clear_all=True) tuf.keydb.clear_keydb(clear_all=True) - # Remove the temporary repository directory, which should contain all the - # metadata, targets, and key files generated of all the test cases - shutil.rmtree(self.temporary_directory) + # Remove top-level temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) From 441a4fa343a04d12d429e4c51ab1d96323a35bce Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Mon, 12 Apr 2021 11:06:19 +0300 Subject: [PATCH 11/17] tests: Remove temp directories after other cleanup Call the parent (Modified_Testcase) tearDown as the last thing in tearDown(). This is good practice anyway and in practice may prevent bugs where the instance needs to cleanup something before Modified_Testcase removes the temp dir. In practice there does not seem to be visible bugs in these tests (as the all have top level temp directory handling in tearDownClass()) Signed-off-by: Jussi Kukkonen --- tests/test_arbitrary_package_attack.py | 6 +++--- tests/test_endless_data_attack.py | 5 ++--- tests/test_extraneous_dependencies_attack.py | 6 +++--- tests/test_indefinite_freeze_attack.py | 6 +++--- tests/test_key_revocation_integration.py | 5 +++-- tests/test_mix_and_match_attack.py | 6 +++--- tests/test_replay_attack.py | 5 ++--- tests/test_updater.py | 5 +++-- tests/test_updater_root_rotation_integration.py | 5 +++-- 9 files changed, 25 insertions(+), 24 deletions(-) diff --git a/tests/test_arbitrary_package_attack.py b/tests/test_arbitrary_package_attack.py index ab68d287a1..02b4c24900 100755 --- a/tests/test_arbitrary_package_attack.py +++ b/tests/test_arbitrary_package_attack.py @@ -134,9 +134,6 @@ def setUp(self): def tearDown(self): - # Modified_TestCase.tearDown() automatically deletes temporary files and - # directories that may have been created during each test case. - unittest_toolbox.Modified_TestCase.tearDown(self) # updater.Updater() populates the roledb with the name "test_repository1" tuf.roledb.clear_roledb(clear_all=True) tuf.keydb.clear_keydb(clear_all=True) @@ -144,6 +141,9 @@ def tearDown(self): # Logs stdout and stderr from the sever subprocess. self.server_process_handler.flush_log() + # Remove temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) + def test_without_tuf(self): diff --git a/tests/test_endless_data_attack.py b/tests/test_endless_data_attack.py index 42bffd1f77..251379bbee 100755 --- a/tests/test_endless_data_attack.py +++ b/tests/test_endless_data_attack.py @@ -135,15 +135,14 @@ def setUp(self): def tearDown(self): - # Modified_TestCase.tearDown() automatically deletes temporary files and - # directories that may have been created during each test case. - unittest_toolbox.Modified_TestCase.tearDown(self) tuf.roledb.clear_roledb(clear_all=True) tuf.keydb.clear_keydb(clear_all=True) # Logs stdout and stderr from the sever subprocess. self.server_process_handler.flush_log() + # Remove temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) def test_without_tuf(self): diff --git a/tests/test_extraneous_dependencies_attack.py b/tests/test_extraneous_dependencies_attack.py index e94a466072..7e2bff7386 100755 --- a/tests/test_extraneous_dependencies_attack.py +++ b/tests/test_extraneous_dependencies_attack.py @@ -141,15 +141,15 @@ def setUp(self): def tearDown(self): - # Modified_TestCase.tearDown() automatically deletes temporary files and - # directories that may have been created during each test case. - unittest_toolbox.Modified_TestCase.tearDown(self) tuf.roledb.clear_roledb(clear_all=True) tuf.keydb.clear_keydb(clear_all=True) # Logs stdout and stderr from the sever subprocess. self.server_process_handler.flush_log() + # Remove temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) + def test_with_tuf(self): # An attacker tries to trick a client into installing an extraneous target diff --git a/tests/test_indefinite_freeze_attack.py b/tests/test_indefinite_freeze_attack.py index acf192704d..71cab0736b 100755 --- a/tests/test_indefinite_freeze_attack.py +++ b/tests/test_indefinite_freeze_attack.py @@ -155,15 +155,15 @@ def setUp(self): def tearDown(self): - # Modified_TestCase.tearDown() automatically deletes temporary files and - # directories that may have been created during each test case. - unittest_toolbox.Modified_TestCase.tearDown(self) tuf.roledb.clear_roledb(clear_all=True) tuf.keydb.clear_keydb(clear_all=True) # Logs stdout and stderr from the sever subprocess. self.server_process_handler.flush_log() + # Remove temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) + def test_without_tuf(self): # Without TUF, Test 1 and Test 2 are functionally equivalent, so we skip diff --git a/tests/test_key_revocation_integration.py b/tests/test_key_revocation_integration.py index bb720f462f..4c00713445 100755 --- a/tests/test_key_revocation_integration.py +++ b/tests/test_key_revocation_integration.py @@ -147,14 +147,15 @@ def setUp(self): def tearDown(self): - # We are inheriting from custom class. - unittest_toolbox.Modified_TestCase.tearDown(self) tuf.roledb.clear_roledb(clear_all=True) tuf.keydb.clear_keydb(clear_all=True) # Logs stdout and stderr from the sever subprocess. self.server_process_handler.flush_log() + # Remove temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) + # UNIT TESTS. def test_timestamp_key_revocation(self): diff --git a/tests/test_mix_and_match_attack.py b/tests/test_mix_and_match_attack.py index 781a03e518..2d9d672abe 100755 --- a/tests/test_mix_and_match_attack.py +++ b/tests/test_mix_and_match_attack.py @@ -140,15 +140,15 @@ def setUp(self): def tearDown(self): - # Modified_TestCase.tearDown() automatically deletes temporary files and - # directories that may have been created during each test case. - unittest_toolbox.Modified_TestCase.tearDown(self) tuf.roledb.clear_roledb(clear_all=True) tuf.keydb.clear_keydb(clear_all=True) # Logs stdout and stderr from the sever subprocess. self.server_process_handler.flush_log() + # Remove temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) + def test_with_tuf(self): # Scenario: diff --git a/tests/test_replay_attack.py b/tests/test_replay_attack.py index 09bf732e87..05cf572c7c 100755 --- a/tests/test_replay_attack.py +++ b/tests/test_replay_attack.py @@ -142,15 +142,14 @@ def setUp(self): def tearDown(self): - # Modified_TestCase.tearDown() automatically deletes temporary files and - # directories that may have been created during each test case. - unittest_toolbox.Modified_TestCase.tearDown(self) tuf.roledb.clear_roledb(clear_all=True) tuf.keydb.clear_keydb(clear_all=True) # Logs stdout and stderr from the sever subprocess. self.server_process_handler.flush_log() + # Remove temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) def test_without_tuf(self): diff --git a/tests/test_updater.py b/tests/test_updater.py index 7a527138dd..0c28e6ca5f 100755 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -182,14 +182,15 @@ def setUp(self): def tearDown(self): - # We are inheriting from custom class. - unittest_toolbox.Modified_TestCase.tearDown(self) tuf.roledb.clear_roledb(clear_all=True) tuf.keydb.clear_keydb(clear_all=True) # Logs stdout and stderr from the sever subprocess. self.server_process_handler.flush_log() + # Remove temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) + # UNIT TESTS. diff --git a/tests/test_updater_root_rotation_integration.py b/tests/test_updater_root_rotation_integration.py index 51f85fd158..8e7e4631c6 100755 --- a/tests/test_updater_root_rotation_integration.py +++ b/tests/test_updater_root_rotation_integration.py @@ -155,14 +155,15 @@ def setUp(self): def tearDown(self): - # We are inheriting from custom class. - unittest_toolbox.Modified_TestCase.tearDown(self) tuf.roledb.clear_roledb(clear_all=True) tuf.keydb.clear_keydb(clear_all=True) # Logs stdout and stderr from the sever subprocess. self.server_process_handler.flush_log() + # Remove temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) + # UNIT TESTS. def test_root_rotation(self): From 1902975d0eb67375061b7e697694557c09f2d823 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Mon, 12 Apr 2021 11:50:53 +0300 Subject: [PATCH 12/17] tests: Kill processes before deleting temp files These tests seem to try to remove temp files before the processes using those files had stopped. This likely lead to an error (and dangling temp files) on Windows, but Modified_Testcase hides the error Make sure temp directories are removed as the last thing in teardown. Signed-off-by: Jussi Kukkonen --- tests/test_download.py | 5 +++-- tests/test_fetcher.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_download.py b/tests/test_download.py index 81e2866407..3b4572ccb0 100755 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -85,13 +85,14 @@ def setUp(self): # Stop server process and perform clean up. def tearDown(self): - unittest_toolbox.Modified_TestCase.tearDown(self) - # Cleans the resources and flush the logged lines (if any). self.server_process_handler.clean() self.target_fileobj.close() + # Remove temp directory + unittest_toolbox.Modified_TestCase.tearDown(self) + # Test: Normal case. def test_download_url_to_tempfileobj(self): diff --git a/tests/test_fetcher.py b/tests/test_fetcher.py index 0df0a4824c..bf94f252d8 100644 --- a/tests/test_fetcher.py +++ b/tests/test_fetcher.py @@ -55,14 +55,15 @@ def setUp(self): # Stop server process and perform clean up. def tearDown(self): - unittest_toolbox.Modified_TestCase.tearDown(self) - # Cleans the resources and flush the logged lines (if any). self.server_process_handler.clean() self.target_fileobj.close() self.temp_file.close() + # Remove temporary directory + unittest_toolbox.Modified_TestCase.tearDown(self) + # Test: Normal case. def test_fetch(self): From f00f89328e368569df04cbba2ad202d1cd5ee359 Mon Sep 17 00:00:00 2001 From: Philippe Coval Date: Wed, 7 Apr 2021 13:02:34 +0200 Subject: [PATCH 13/17] tests: Use current python interpreter for sub tests Can be useful to run tests using distro runtimes, (like python3 on Debian). Relate-to: https://github.com/theupdateframework/tuf/issues/263 Origin: https://salsa.debian.org/rzr/python-tuf/-/tree/debian/review/master Forwarded: https://github.com/theupdateframework/tuf/pull/1337 Signed-off-by: Philippe Coval --- tests/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index caccb18086..6a3ee66b1d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -25,6 +25,7 @@ import errno import logging import socket +import sys import time import subprocess import threading @@ -186,7 +187,7 @@ def _start_process(self, extra_cmd_args, popen_cwd): """Starts the process running the server.""" # The "-u" option forces stdin, stdout and stderr to be unbuffered. - command = ['python', '-u', self.server] + extra_cmd_args + command = [sys.executable, '-u', self.server] + extra_cmd_args # Reusing one subprocess in multiple tests, but split up the logs for each. self.__server_process = subprocess.Popen(command, From 1eaef0093b0a74ecc8c612356c0741296fe4a23a Mon Sep 17 00:00:00 2001 From: Velichka Atanasova Date: Mon, 12 Apr 2021 13:57:01 +0300 Subject: [PATCH 14/17] Add is_expired method to the Signed class Checks metadata expiration against a reference time (a naive datetime in UTC). If not provided, checks against the current UTC date and time. Returns True if expiration time is less than the reference time. Signed-off-by: Velichka Atanasova --- tests/test_api.py | 20 +++++++++++++++++++- tuf/api/metadata.py | 16 ++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index 24f6ace24a..c47e0ef34c 100755 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -219,7 +219,25 @@ def test_metadata_base(self): md.signed.bump_expiration(timedelta(days=365)) self.assertEqual(md.signed.expires, datetime(2031, 1, 2, 0, 0)) - + # Test is_expired with reference_time provided + is_expired = md.signed.is_expired(md.signed.expires) + self.assertTrue(is_expired) + is_expired = md.signed.is_expired(md.signed.expires + timedelta(days=1)) + self.assertTrue(is_expired) + is_expired = md.signed.is_expired(md.signed.expires - timedelta(days=1)) + self.assertFalse(is_expired) + + # Test is_expired without reference_time, + # manipulating md.signed.expires + expires = md.signed.expires + md.signed.expires = datetime.utcnow() + is_expired = md.signed.is_expired() + self.assertTrue(is_expired) + md.signed.expires = datetime.utcnow() + timedelta(days=1) + is_expired = md.signed.is_expired() + self.assertFalse(is_expired) + md.signed.expires = expires + def test_metadata_snapshot(self): snapshot_path = os.path.join( self.repo_dir, 'metadata', 'snapshot.json') diff --git a/tuf/api/metadata.py b/tuf/api/metadata.py index 99efd9b510..f579fd669c 100644 --- a/tuf/api/metadata.py +++ b/tuf/api/metadata.py @@ -351,6 +351,22 @@ def _common_fields_to_dict(self) -> Dict[str, Any]: "expires": self.expires.isoformat() + "Z", } + def is_expired(self, reference_time: datetime = None) -> bool: + """Checks metadata expiration against a reference time. + + Args: + reference_time: Optional; The time to check expiration date against. + A naive datetime in UTC expected. + If not provided, checks against the current UTC date and time. + + Returns: + True if expiration time is less than the reference time. + """ + if reference_time is None: + reference_time = datetime.utcnow() + + return reference_time >= self.expires + # Modification. def bump_expiration(self, delta: timedelta = timedelta(days=1)) -> None: """Increments the expires attribute by the passed timedelta. """ From d0fa8fc8caf2b71c1cd6abf32532c02eb2d61a5a Mon Sep 17 00:00:00 2001 From: Martin Vrachev Date: Thu, 8 Apr 2021 19:40:23 +0300 Subject: [PATCH 15/17] Document ADR 0008 about unrecognized fields Even though, this ADR documents something already implied in the TUF spec in [document formats](https://theupdateframework.github.io/specification/latest/#document-formats) it seems better to document this decision clearly so that it could be referenced and give an explanation why someone can load a metadata file with additional unrecognized fields. Signed-off-by: Martin Vrachev --- docs/adr/0008-accept-unrecognised-fields.md | 51 +++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 docs/adr/0008-accept-unrecognised-fields.md diff --git a/docs/adr/0008-accept-unrecognised-fields.md b/docs/adr/0008-accept-unrecognised-fields.md new file mode 100644 index 0000000000..9b633a2fc7 --- /dev/null +++ b/docs/adr/0008-accept-unrecognised-fields.md @@ -0,0 +1,51 @@ +# Accept metadata that includes unrecognized fields + +- Status: accepted +- Date: 2021-04-08 + +Technical Story: https://github.com/theupdateframework/tuf/issues/1266 + +## Context and Problem Statement + +The current reference implementation will ignore unrecognized fields in a +metadata file when loading it. +This leads to the side effect that if you read a metadata file with unrecognized +fields and immediately write it back to the disk, this file will be modified. + +Furthermore, some TAPs like: +- [TAP 6](https://github.com/theupdateframework/taps/blob/master/tap6.md) +- [TAP 10](https://github.com/theupdateframework/taps/blob/master/tap10.md) +- [TAP 14](https://github.com/theupdateframework/taps/blob/master/tap14.md) +- [TAP 15](https://github.com/theupdateframework/taps/blob/master/tap15.md) +- [TAP 16](https://github.com/theupdateframework/taps/blob/master/tap16.md) + +are relying on that unrecognized fields will be accepted to introduce new fields +to the specification without making the metadata invalid for older clients who +don't recognize the field. + +## Decision Drivers +- The TUF specification implies support for unrecognized attribute-value fields, +see [Document formats](https://theupdateframework.github.io/specification/latest/#document-formats) +- If we perform the following operations on a metadata file with no +intermediate operations: +1. read the metadata file +2. write the metadata file back to the disk + +then, the checksum (the content) of the file must not be changed. +- Flexibility to add new fields in the spec without adding breaking changes. + +## Considered Options +- Ignore and drop unrecognized fields. +- Ignore, but store unrecognized fields as an additional attribute. + +## Decision Outcome + +Chosen option: "Ignore, but store unrecognized fields as an additional +attribute." +The motivation for this decision is that the TUF specification already implies +that we should accept unrecognized fields for backward compatibility and easier +future extensibility. + +Additionally, it seems unacceptable to change a metadata file content just by +reading and writing it back. + From f695bfd24eebc96fb6cc29c6146d62b8efcc0439 Mon Sep 17 00:00:00 2001 From: Martin Vrachev Date: Fri, 16 Apr 2021 12:26:42 +0300 Subject: [PATCH 16/17] Add ADR8 to the ADR's index file Signed-off-by: Martin Vrachev --- docs/adr/index.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/adr/index.md b/docs/adr/index.md index 914c681b96..97b722afe9 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -11,6 +11,9 @@ This log lists the architectural decisions for tuf. - [ADR-0004](0004-extent-of-OOP-in-metadata-model.md) - Add classes for complex metadata attributes - [ADR-0005](0005-use-google-python-style-guide.md) - Use Google Python style guide with minimal refinements + +- [ADR-0008](0008-accept-unrecognised-fields.md) - Accept metadata that includes unrecognized fields + For new ADRs, please use [template.md](template.md) as basis. From 4e8738ffa5a8c7f324443a41e767d853f0ed1215 Mon Sep 17 00:00:00 2001 From: Jussi Kukkonen Date: Fri, 16 Apr 2021 16:12:15 +0300 Subject: [PATCH 17/17] api: Add Metadata.from_bytes() This is essentially short-hand for JSONDeserializer().deserialize(data) but seems much easier for the API user so may be worth it. Metadata.from_file() now uses Metadata.from_bytes() internally. Signed-off-by: Jussi Kukkonen --- tests/test_api.py | 9 +++++---- tuf/api/metadata.py | 34 +++++++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index c47e0ef34c..544937ef36 100755 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -105,8 +105,7 @@ def test_generic_read(self): path = os.path.join(self.repo_dir, 'metadata', metadata + '.json') metadata_obj = Metadata.from_file(path) with open(path, 'rb') as f: - metadata_str = f.read() - metadata_obj2 = JSONDeserializer().deserialize(metadata_str) + metadata_obj2 = Metadata.from_bytes(f.read()) # Assert that both methods instantiate the right inner class for # each metadata type and ... @@ -119,15 +118,17 @@ def test_generic_read(self): self.assertDictEqual( metadata_obj.to_dict(), metadata_obj2.to_dict()) - # Assert that it chokes correctly on an unknown metadata type bad_metadata_path = 'bad-metadata.json' bad_metadata = {'signed': {'_type': 'bad-metadata'}} + bad_string = json.dumps(bad_metadata).encode('utf-8') with open(bad_metadata_path, 'wb') as f: - f.write(json.dumps(bad_metadata).encode('utf-8')) + f.write(bad_string) with self.assertRaises(DeserializationError): Metadata.from_file(bad_metadata_path) + with self.assertRaises(DeserializationError): + Metadata.from_bytes(bad_string) os.remove(bad_metadata_path) diff --git a/tuf/api/metadata.py b/tuf/api/metadata.py index f579fd669c..86d1b7df9e 100644 --- a/tuf/api/metadata.py +++ b/tuf/api/metadata.py @@ -128,6 +128,32 @@ def from_file( A TUF Metadata object. """ + if storage_backend is None: + storage_backend = FilesystemBackend() + + with storage_backend.get(filename) as file_obj: + return cls.from_bytes(file_obj.read(), deserializer) + + @staticmethod + def from_bytes( + data: bytes, + deserializer: Optional[MetadataDeserializer] = None, + ) -> "Metadata": + """Loads TUF metadata from raw data. + + Arguments: + data: metadata content as bytes. + deserializer: Optional; A MetadataDeserializer instance that + implements deserialization. Default is JSONDeserializer. + + Raises: + tuf.api.serialization.DeserializationError: + The file cannot be deserialized. + + Returns: + A TUF Metadata object. + """ + if deserializer is None: # Use local scope import to avoid circular import errors # pylint: disable=import-outside-toplevel @@ -135,13 +161,7 @@ def from_file( deserializer = JSONDeserializer() - if storage_backend is None: - storage_backend = FilesystemBackend() - - with storage_backend.get(filename) as file_obj: - raw_data = file_obj.read() - - return deserializer.deserialize(raw_data) + return deserializer.deserialize(data) def to_dict(self) -> Dict[str, Any]: """Returns the dict representation of self. """