From 35587c4a09102d6ee7a15aa7b38ecae677b9683c Mon Sep 17 00:00:00 2001 From: Teodora Sechkova Date: Thu, 25 Mar 2021 16:38:03 +0200 Subject: [PATCH 01/11] Move download modules inside client directory The modules performing network download are used only by the client side of TUF. Move them inside the client directory for the refactored client. Move the _mirror_*download functions from Updater to mirrors.py. Signed-off-by: Teodora Sechkova --- tuf/client_rework/download.py | 323 ++++++++++++++++++++++++++ tuf/client_rework/fetcher.py | 38 +++ tuf/client_rework/mirrors.py | 194 ++++++++++++++++ tuf/client_rework/requests_fetcher.py | 173 ++++++++++++++ tuf/client_rework/updater_rework.py | 83 ++----- 5 files changed, 750 insertions(+), 61 deletions(-) create mode 100644 tuf/client_rework/download.py create mode 100644 tuf/client_rework/fetcher.py create mode 100644 tuf/client_rework/mirrors.py create mode 100644 tuf/client_rework/requests_fetcher.py diff --git a/tuf/client_rework/download.py b/tuf/client_rework/download.py new file mode 100644 index 0000000000..2d946ef891 --- /dev/null +++ b/tuf/client_rework/download.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python + +# Copyright 2012 - 2017, New York University and the TUF contributors +# SPDX-License-Identifier: MIT OR Apache-2.0 + +""" + + download.py + + + February 21, 2012. Based on previous version by Geremy Condra. + + + Konstantin Andrianov + Vladimir Diaz + + + See LICENSE-MIT OR LICENSE for licensing information. + + + Download metadata and target files and check their validity. The hash and + length of a downloaded file has to match the hash and length supplied by the + metadata of that file. +""" + +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +import logging +import timeit +import tempfile + +import securesystemslib +import securesystemslib.util +import six + +import tuf +import tuf.exceptions +import tuf.formats + +# See 'log.py' to learn how logging is handled in TUF. +logger = logging.getLogger(__name__) + + +def safe_download(url, required_length, fetcher): + """ + + Given the 'url' and 'required_length' of the desired file, open a connection + to 'url', download it, and return the contents of the file. Also ensure + the length of the downloaded file matches 'required_length' exactly. + tuf.download.unsafe_download() may be called if an upper download limit is + preferred. + + + url: + A URL string that represents the location of the file. + + required_length: + An integer value representing the length of the file. This is an exact + limit. + + fetcher: + An object implementing FetcherInterface that performs the network IO + operations. + + + A file object is created on disk to store the contents of 'url'. + + + tuf.ssl_commons.exceptions.DownloadLengthMismatchError, if there was a + mismatch of observed vs expected lengths while downloading the file. + + securesystemslib.exceptions.FormatError, if any of the arguments are + improperly formatted. + + Any other unforeseen runtime exception. + + + A file object that points to the contents of 'url'. + """ + + # Do all of the arguments have the appropriate format? + # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. + securesystemslib.formats.URL_SCHEMA.check_match(url) + tuf.formats.LENGTH_SCHEMA.check_match(required_length) + + return _download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True) + + + + + +def unsafe_download(url, required_length, fetcher): + """ + + Given the 'url' and 'required_length' of the desired file, open a connection + to 'url', download it, and return the contents of the file. Also ensure + the length of the downloaded file is up to 'required_length', and no larger. + tuf.download.safe_download() may be called if an exact download limit is + preferred. + + + url: + A URL string that represents the location of the file. + + required_length: + An integer value representing the length of the file. This is an upper + limit. + + fetcher: + An object implementing FetcherInterface that performs the network IO + operations. + + + A file object is created on disk to store the contents of 'url'. + + + tuf.ssl_commons.exceptions.DownloadLengthMismatchError, if there was a + mismatch of observed vs expected lengths while downloading the file. + + securesystemslib.exceptions.FormatError, if any of the arguments are + improperly formatted. + + Any other unforeseen runtime exception. + + + A file object that points to the contents of 'url'. + """ + + # Do all of the arguments have the appropriate format? + # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. + securesystemslib.formats.URL_SCHEMA.check_match(url) + tuf.formats.LENGTH_SCHEMA.check_match(required_length) + + return _download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=False) + + + + + +def _download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True): + """ + + Given the url and length of the desired file, this function opens a + connection to 'url' and downloads the file while ensuring its length + matches 'required_length' if 'STRICT_REQUIRED_LENGH' is True (If False, + the file's length is not checked and a slow retrieval exception is raised + if the downloaded rate falls below the acceptable rate). + + + url: + A URL string that represents the location of the file. + + required_length: + An integer value representing the length of the file. + + STRICT_REQUIRED_LENGTH: + A Boolean indicator used to signal whether we should perform strict + checking of required_length. True by default. We explicitly set this to + False when we know that we want to turn this off for downloading the + timestamp metadata, which has no signed required_length. + + + A file object is created on disk to store the contents of 'url'. + + + tuf.exceptions.DownloadLengthMismatchError, if there was a + mismatch of observed vs expected lengths while downloading the file. + + securesystemslib.exceptions.FormatError, if any of the arguments are + improperly formatted. + + Any other unforeseen runtime exception. + + + A file object that points to the contents of 'url'. + """ + # 'url.replace('\\', '/')' is needed for compatibility with Windows-based + # systems, because they might use back-slashes in place of forward-slashes. + # This converts it to the common format. unquote() replaces %xx escapes in a + # url with their single-character equivalent. A back-slash may be encoded as + # %5c in the url, which should also be replaced with a forward slash. + url = six.moves.urllib.parse.unquote(url).replace('\\', '/') + logger.info('Downloading: ' + repr(url)) + + # This is the temporary file that we will return to contain the contents of + # the downloaded file. + temp_file = tempfile.TemporaryFile() + + average_download_speed = 0 + number_of_bytes_received = 0 + + try: + chunks = fetcher.fetch(url, required_length) + start_time = timeit.default_timer() + for chunk in chunks: + + stop_time = timeit.default_timer() + temp_file.write(chunk) + + # Measure the average download speed. + number_of_bytes_received += len(chunk) + seconds_spent_receiving = stop_time - start_time + average_download_speed = number_of_bytes_received / seconds_spent_receiving + + if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: + logger.debug('The average download speed dropped below the minimum' + ' average download speed set in tuf.settings.py. Stopping the' + ' download!') + break + + else: + logger.debug('The average download speed has not dipped below the' + ' minimum average download speed set in tuf.settings.py.') + + # Does the total number of downloaded bytes match the required length? + _check_downloaded_length(number_of_bytes_received, required_length, + STRICT_REQUIRED_LENGTH=STRICT_REQUIRED_LENGTH, + average_download_speed=average_download_speed) + + except Exception: + # Close 'temp_file'. Any written data is lost. + temp_file.close() + logger.debug('Could not download URL: ' + repr(url)) + raise + + else: + return temp_file + + + + +def _check_downloaded_length(total_downloaded, required_length, + STRICT_REQUIRED_LENGTH=True, + average_download_speed=None): + """ + + A helper function which checks whether the total number of downloaded bytes + matches our expectation. + + + total_downloaded: + The total number of bytes supposedly downloaded for the file in question. + + required_length: + The total number of bytes expected of the file as seen from its metadata. + The Timestamp role is always downloaded without a known file length, and + the Root role when the client cannot download any of the required + top-level roles. In both cases, 'required_length' is actually an upper + limit on the length of the downloaded file. + + STRICT_REQUIRED_LENGTH: + A Boolean indicator used to signal whether we should perform strict + checking of required_length. True by default. We explicitly set this to + False when we know that we want to turn this off for downloading the + timestamp metadata, which has no signed required_length. + + average_download_speed: + The average download speed for the downloaded file. + + + None. + + + securesystemslib.exceptions.DownloadLengthMismatchError, if + STRICT_REQUIRED_LENGTH is True and total_downloaded is not equal + required_length. + + tuf.exceptions.SlowRetrievalError, if the total downloaded was + done in less than the acceptable download speed (as set in + tuf.settings.py). + + + None. + """ + + if total_downloaded == required_length: + logger.info('Downloaded ' + str(total_downloaded) + ' bytes out of the' + ' expected ' + str(required_length) + ' bytes.') + + else: + difference_in_bytes = abs(total_downloaded - required_length) + + # What we downloaded is not equal to the required length, but did we ask + # for strict checking of required length? + if STRICT_REQUIRED_LENGTH: + logger.info('Downloaded ' + str(total_downloaded) + ' bytes, but' + ' expected ' + str(required_length) + ' bytes. There is a difference' + ' of ' + str(difference_in_bytes) + ' bytes.') + + # If the average download speed is below a certain threshold, we flag + # this as a possible slow-retrieval attack. + logger.debug('Average download speed: ' + repr(average_download_speed)) + logger.debug('Minimum average download speed: ' + repr(tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED)) + + if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: + raise tuf.exceptions.SlowRetrievalError(average_download_speed) + + else: + logger.debug('Good average download speed: ' + + repr(average_download_speed) + ' bytes per second') + + raise tuf.exceptions.DownloadLengthMismatchError(required_length, total_downloaded) + + else: + # We specifically disabled strict checking of required length, but we + # will log a warning anyway. This is useful when we wish to download the + # Timestamp or Root metadata, for which we have no signed metadata; so, + # we must guess a reasonable required_length for it. + if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: + raise tuf.exceptions.SlowRetrievalError(average_download_speed) + + else: + logger.debug('Good average download speed: ' + + repr(average_download_speed) + ' bytes per second') + + logger.info('Downloaded ' + str(total_downloaded) + ' bytes out of an' + ' upper limit of ' + str(required_length) + ' bytes.') diff --git a/tuf/client_rework/fetcher.py b/tuf/client_rework/fetcher.py new file mode 100644 index 0000000000..8768bdd4b9 --- /dev/null +++ b/tuf/client_rework/fetcher.py @@ -0,0 +1,38 @@ +# Copyright 2021, New York University and the TUF contributors +# SPDX-License-Identifier: MIT OR Apache-2.0 + +"""Provides an interface for network IO abstraction. +""" + +# Imports +import abc + +# Classes +class FetcherInterface(): + """Defines an interface for abstract network download. + + By providing a concrete implementation of the abstract interface, + users of the framework can plug-in their preferred/customized + network stack. + """ + + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def fetch(self, url, required_length): + """Fetches the contents of HTTP/HTTPS url from a remote server. + + Ensures the length of the downloaded data is up to 'required_length'. + + Arguments: + url: A URL string that represents a file location. + required_length: An integer value representing the file length in bytes. + + Raises: + tuf.exceptions.SlowRetrievalError: A timeout occurs while receiving data. + tuf.exceptions.FetcherHTTPError: An HTTP error code is received. + + Returns: + A bytes iterator + """ + raise NotImplementedError # pragma: no cover diff --git a/tuf/client_rework/mirrors.py b/tuf/client_rework/mirrors.py new file mode 100644 index 0000000000..a9e4dd266b --- /dev/null +++ b/tuf/client_rework/mirrors.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python + +# Copyright 2012 - 2017, New York University and the TUF contributors +# SPDX-License-Identifier: MIT OR Apache-2.0 + +""" + + mirrors.py + + + Konstantin Andrianov. + Derived from original mirrors.py written by Geremy Condra. + + + March 12, 2012. + + + See LICENSE-MIT OR LICENSE for licensing information. + + + Extract a list of mirror urls corresponding to the file type and the location + of the file with respect to the base url. +""" + +# Help with Python 3 compatibility, where the print statement is a function, an +# implicit relative import is invalid, and the '/' operator performs true +# division. Example: print 'hello world' raises a 'SyntaxError' exception. +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from typing import TextIO, BinaryIO, Dict + +import os + +import tuf +import tuf.formats +import tuf.client_rework.download as download + +import securesystemslib +import six + +# The type of file to be downloaded from a repository. The +# 'get_list_of_mirrors' function supports these file types. +_SUPPORTED_FILE_TYPES = ['meta', 'target'] + + +def get_list_of_mirrors(file_type, file_path, mirrors_dict): + """ + + Get a list of mirror urls from a mirrors dictionary, provided the type + and the path of the file with respect to the base url. + + + file_type: + Type of data needed for download, must correspond to one of the strings + in the list ['meta', 'target']. 'meta' for metadata file type or + 'target' for target file type. It should correspond to + NAME_SCHEMA format. + + file_path: + A relative path to the file that corresponds to RELPATH_SCHEMA format. + Ex: 'http://url_prefix/targets_path/file_path' + + mirrors_dict: + A mirrors_dict object that corresponds to MIRRORDICT_SCHEMA, where + keys are strings and values are MIRROR_SCHEMA. An example format + of MIRROR_SCHEMA: + + {'url_prefix': 'http://localhost:8001', + 'metadata_path': 'metadata/', + 'targets_path': 'targets/', + 'confined_target_dirs': ['targets/snapshot1/', ...], + 'custom': {...}} + + The 'custom' field is optional. + + + securesystemslib.exceptions.Error, on unsupported 'file_type'. + + securesystemslib.exceptions.FormatError, on bad argument. + + + List of mirror urls corresponding to the file_type and file_path. If no + match is found, empty list is returned. + """ + + # Checking if all the arguments have appropriate format. + tuf.formats.RELPATH_SCHEMA.check_match(file_path) + tuf.formats.MIRRORDICT_SCHEMA.check_match(mirrors_dict) + securesystemslib.formats.NAME_SCHEMA.check_match(file_type) + + # Verify 'file_type' is supported. + if file_type not in _SUPPORTED_FILE_TYPES: + raise securesystemslib.exceptions.Error('Invalid file_type argument.' + ' Supported file types: ' + repr(_SUPPORTED_FILE_TYPES)) + path_key = 'metadata_path' if file_type == 'meta' else 'targets_path' + + # Reference to 'securesystemslib.util.file_in_confined_directories()' (improve + # readability). This function checks whether a mirror should serve a file to + # the client. A client may be confined to certain paths on a repository + # mirror when fetching target files. This field may be set by the client + # when the repository mirror is added to the 'tuf.client.updater.Updater' + # object. + in_confined_directory = securesystemslib.util.file_in_confined_directories + + list_of_mirrors = [] + for junk, mirror_info in six.iteritems(mirrors_dict): + # Does mirror serve this file type at all? + path = mirror_info.get(path_key) + if path is None: + continue + + # for targets, ensure directory confinement + if path_key == 'targets_path': + full_filepath = os.path.join(path, file_path) + confined_target_dirs = mirror_info.get('confined_target_dirs') + # confined_target_dirs is an optional field + if confined_target_dirs and not in_confined_directory(full_filepath, + confined_target_dirs): + continue + + # urllib.quote(string) replaces special characters in string using the %xx + # escape. This is done to avoid parsing issues of the URL on the server + # side. Do *NOT* pass URLs with Unicode characters without first encoding + # the URL as UTF-8. We need a long-term solution with #61. + # http://bugs.python.org/issue1712522 + file_path = six.moves.urllib.parse.quote(file_path) + url = os.path.join(mirror_info['url_prefix'], path, file_path) + + # The above os.path.join() result as well as input file_path may be + # invalid on windows (might contain both separator types), see #1077. + # Make sure the URL doesn't contain backward slashes on Windows. + list_of_mirrors.append(url.replace('\\', '/')) + + return list_of_mirrors + + +def _mirror_meta_download(filename: str, upper_length: int, + mirrors_config: Dict, + fetcher: "FetcherInterface") -> TextIO: + """ + Download metadata file from the list of metadata mirrors + """ + file_mirrors = get_list_of_mirrors('meta', filename, mirrors_config) + + file_mirror_errors = {} + for file_mirror in file_mirrors: + try: + temp_obj = download.unsafe_download( + file_mirror, + upper_length, + fetcher) + + temp_obj.seek(0) + yield temp_obj + + except Exception as exception: + file_mirror_errors[file_mirror] = exception + + finally: + if file_mirror_errors: + raise tuf.exceptions.NoWorkingMirrorError( + file_mirror_errors) + + +def _mirror_target_download(fileinfo: str, mirrors_config: Dict, + fetcher: "FetcherInterface") -> BinaryIO: + """ + Download target file from the list of target mirrors + """ + # full_filename = _get_full_name(filename) + file_mirrors = get_list_of_mirrors('target', fileinfo['filepath'], + mirrors_config) + + file_mirror_errors = {} + for file_mirror in file_mirrors: + try: + temp_obj = download.safe_download( + file_mirror, + fileinfo['fileinfo']['length'], + fetcher) + + temp_obj.seek(0) + yield temp_obj + + except Exception as exception: + file_mirror_errors[file_mirror] = exception + + finally: + if file_mirror_errors: + raise tuf.exceptions.NoWorkingMirrorError( + file_mirror_errors) diff --git a/tuf/client_rework/requests_fetcher.py b/tuf/client_rework/requests_fetcher.py new file mode 100644 index 0000000000..8074890d25 --- /dev/null +++ b/tuf/client_rework/requests_fetcher.py @@ -0,0 +1,173 @@ +# Copyright 2021, New York University and the TUF contributors +# SPDX-License-Identifier: MIT OR Apache-2.0 + +"""Provides an implementation of FetcherInterface using the Requests HTTP + library. +""" + +# Imports +import requests +import six +import logging +import time + +import urllib3.exceptions + +import tuf.exceptions +import tuf.settings + +from tuf.client_rework.fetcher import FetcherInterface + +# Globals +logger = logging.getLogger(__name__) + +# Classess +class RequestsFetcher(FetcherInterface): + """A concrete implementation of FetcherInterface based on the Requests + library. + + Attributes: + _sessions: A dictionary of Requests.Session objects storing a separate + session per scheme+hostname combination. + """ + + def __init__(self): + # From http://docs.python-requests.org/en/master/user/advanced/#session-objects: + # + # "The Session object allows you to persist certain parameters across + # requests. It also persists cookies across all requests made from the + # Session instance, and will use urllib3's connection pooling. So if you're + # making several requests to the same host, the underlying TCP connection + # will be reused, which can result in a significant performance increase + # (see HTTP persistent connection)." + # + # NOTE: We use a separate requests.Session per scheme+hostname combination, + # in order to reuse connections to the same hostname to improve efficiency, + # but avoiding sharing state between different hosts-scheme combinations to + # minimize subtle security issues. Some cookies may not be HTTP-safe. + self._sessions = {} + + + def fetch(self, url, required_length): + """Fetches the contents of HTTP/HTTPS url from a remote server. + + Ensures the length of the downloaded data is up to 'required_length'. + + Arguments: + url: A URL string that represents a file location. + required_length: An integer value representing the file length in bytes. + + Raises: + tuf.exceptions.SlowRetrievalError: A timeout occurs while receiving data. + tuf.exceptions.FetcherHTTPError: An HTTP error code is received. + + Returns: + A bytes iterator + """ + # Get a customized session for each new schema+hostname combination. + session = self._get_session(url) + + # Get the requests.Response object for this URL. + # + # Defer downloading the response body with stream=True. + # Always set the timeout. This timeout value is interpreted by requests as: + # - connect timeout (max delay before first byte is received) + # - read (gap) timeout (max delay between bytes received) + response = session.get(url, stream=True, + timeout=tuf.settings.SOCKET_TIMEOUT) + # Check response status. + try: + response.raise_for_status() + except requests.HTTPError as e: + response.close() + status = e.response.status_code + raise tuf.exceptions.FetcherHTTPError(str(e), status) + + + # Define a generator function to be returned by fetch. This way the caller + # of fetch can differentiate between connection and actual data download + # and measure download times accordingly. + def chunks(): + try: + bytes_received = 0 + while True: + # We download a fixed chunk of data in every round. This is so that we + # can defend against slow retrieval attacks. Furthermore, we do not + # wish to download an extremely large file in one shot. + # Before beginning the round, sleep (if set) for a short amount of + # time so that the CPU is not hogged in the while loop. + if tuf.settings.SLEEP_BEFORE_ROUND: + time.sleep(tuf.settings.SLEEP_BEFORE_ROUND) + + read_amount = min( + tuf.settings.CHUNK_SIZE, required_length - bytes_received) + + # NOTE: This may not handle some servers adding a Content-Encoding + # header, which may cause urllib3 to misbehave: + # https://github.com/pypa/pip/blob/404838abcca467648180b358598c597b74d568c9/src/pip/_internal/download.py#L547-L582 + data = response.raw.read(read_amount) + bytes_received += len(data) + + # We might have no more data to read. Check number of bytes downloaded. + if not data: + logger.debug('Downloaded ' + repr(bytes_received) + '/' + + repr(required_length) + ' bytes.') + + # Finally, we signal that the download is complete. + break + + yield data + + if bytes_received >= required_length: + break + + except urllib3.exceptions.ReadTimeoutError as e: + raise tuf.exceptions.SlowRetrievalError(str(e)) + + finally: + response.close() + + return chunks() + + + + def _get_session(self, url): + """Returns a different customized requests.Session per schema+hostname + combination. + """ + # Use a different requests.Session per schema+hostname combination, to + # reuse connections while minimizing subtle security issues. + parsed_url = six.moves.urllib.parse.urlparse(url) + + if not parsed_url.scheme or not parsed_url.hostname: + raise tuf.exceptions.URLParsingError( + 'Could not get scheme and hostname from URL: ' + url) + + session_index = parsed_url.scheme + '+' + parsed_url.hostname + + logger.debug('url: ' + url) + logger.debug('session index: ' + session_index) + + session = self._sessions.get(session_index) + + if not session: + session = requests.Session() + self._sessions[session_index] = session + + # Attach some default headers to every Session. + requests_user_agent = session.headers['User-Agent'] + # Follows the RFC: https://tools.ietf.org/html/rfc7231#section-5.5.3 + tuf_user_agent = 'tuf/' + tuf.__version__ + ' ' + requests_user_agent + session.headers.update({ + # Tell the server not to compress or modify anything. + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding#Directives + 'Accept-Encoding': 'identity', + # The TUF user agent. + 'User-Agent': tuf_user_agent}) + + logger.debug('Made new session for ' + session_index) + + else: + logger.debug('Reusing session for ' + session_index) + + return session diff --git a/tuf/client_rework/updater_rework.py b/tuf/client_rework/updater_rework.py index c7820a4ab4..8ddde1e4eb 100644 --- a/tuf/client_rework/updater_rework.py +++ b/tuf/client_rework/updater_rework.py @@ -16,8 +16,9 @@ from securesystemslib import hash as sslib_hash from securesystemslib import util as sslib_util -from tuf import download, exceptions, mirrors, requests_fetcher, settings +from tuf import exceptions, settings from tuf.client.fetcher import FetcherInterface +from tuf.client_rework import download, mirrors, requests_fetcher from .metadata_wrapper import ( RootWrapper, @@ -146,9 +147,11 @@ def download_target(self, target: Dict, destination_directory: str): This method performs the actual download of the specified target. The file is saved to the 'destination_directory' argument. """ - try: - for temp_obj in self._mirror_target_download(target): + for temp_obj in mirrors._mirror_target_download( + target, self._mirrors, self._fetcher + ): + self._verify_target_file(temp_obj, target) # break? should we break after first successful download? @@ -161,58 +164,6 @@ def download_target(self, target: Dict, destination_directory: str): # TODO: do something with exceptions raise - def _mirror_meta_download(self, filename: str, upper_length: int) -> TextIO: - """ - Download metadata file from the list of metadata mirrors - """ - file_mirrors = mirrors.get_list_of_mirrors( - "meta", filename, self._mirrors - ) - - file_mirror_errors = {} - for file_mirror in file_mirrors: - try: - temp_obj = download.unsafe_download( - file_mirror, upper_length, self._fetcher - ) - - temp_obj.seek(0) - yield temp_obj - - # pylint: disable=broad-except - except Exception as exception: - file_mirror_errors[file_mirror] = exception - - finally: - if file_mirror_errors: - raise exceptions.NoWorkingMirrorError(file_mirror_errors) - - def _mirror_target_download(self, fileinfo: str) -> BinaryIO: - """ - Download target file from the list of target mirrors - """ - # full_filename = _get_full_name(filename) - file_mirrors = mirrors.get_list_of_mirrors( - "target", fileinfo["filepath"], self._mirrors - ) - - file_mirror_errors = {} - for file_mirror in file_mirrors: - try: - temp_obj = download.safe_download( - file_mirror, fileinfo["fileinfo"]["length"], self._fetcher - ) - - temp_obj.seek(0) - yield temp_obj - # pylint: disable=broad-except - except Exception as exception: - file_mirror_errors[file_mirror] = exception - - finally: - if file_mirror_errors: - raise exceptions.NoWorkingMirrorError(file_mirror_errors) - def _get_full_meta_name( self, role: str, extension: str = ".json", version: int = None ) -> str: @@ -266,9 +217,11 @@ def _load_root(self) -> None: verified_root = None for next_version in range(lower_bound, upper_bound): try: - mirror_download = self._mirror_meta_download( + mirror_download = mirrors._mirror_meta_download( self._get_relative_meta_name("root", version=next_version), settings.DEFAULT_ROOT_REQUIRED_LENGTH, + self._mirrors, + self._fetcher, ) for temp_obj in mirror_download: @@ -327,9 +280,13 @@ def _load_timestamp(self) -> None: TODO """ # TODO Check if timestamp exists locally - for temp_obj in self._mirror_meta_download( - "timestamp.json", settings.DEFAULT_TIMESTAMP_REQUIRED_LENGTH + for temp_obj in mirrors._mirror_meta_download( + "timestamp.json", + settings.DEFAULT_TIMESTAMP_REQUIRED_LENGTH, + self._mirrors, + self._fetcher, ): + try: verified_tampstamp = self._verify_timestamp(temp_obj) # break? should we break after first successful download? @@ -364,7 +321,10 @@ def _load_snapshot(self) -> None: # Check if exists locally # self.loadLocal('snapshot', snapshotVerifier) - for temp_obj in self._mirror_meta_download("snapshot.json", length): + for temp_obj in mirrors._mirror_meta_download( + "snapshot.json", length, self._mirrors, self._fetcher + ): + try: verified_snapshot = self._verify_snapshot(temp_obj) # break? should we break after first successful download? @@ -400,9 +360,10 @@ def _load_targets(self, targets_role: str, parent_role: str) -> None: # Check if exists locally # self.loadLocal('snapshot', targetsVerifier) - for temp_obj in self._mirror_meta_download( - targets_role + ".json", length + for temp_obj in mirrors._mirror_meta_download( + targets_role + ".json", length, self._mirrors, self._fetcher ): + try: verified_targets = self._verify_targets( temp_obj, targets_role, parent_role From 81747edf97ff03fbe028fc993c025bcabffb8b77 Mon Sep 17 00:00:00 2001 From: Teodora Sechkova Date: Wed, 31 Mar 2021 12:38:06 +0300 Subject: [PATCH 02/11] Remove (un)safe_download functions The two functions safe/unsafe_download differ only by setting a single boolean flag. Remove them and call directly _download_file instead. Signed-off-by: Teodora Sechkova --- tuf/client_rework/download.py | 103 ++-------------------------------- tuf/client_rework/mirrors.py | 7 ++- 2 files changed, 10 insertions(+), 100 deletions(-) diff --git a/tuf/client_rework/download.py b/tuf/client_rework/download.py index 2d946ef891..4cc855f372 100644 --- a/tuf/client_rework/download.py +++ b/tuf/client_rework/download.py @@ -47,103 +47,7 @@ logger = logging.getLogger(__name__) -def safe_download(url, required_length, fetcher): - """ - - Given the 'url' and 'required_length' of the desired file, open a connection - to 'url', download it, and return the contents of the file. Also ensure - the length of the downloaded file matches 'required_length' exactly. - tuf.download.unsafe_download() may be called if an upper download limit is - preferred. - - - url: - A URL string that represents the location of the file. - - required_length: - An integer value representing the length of the file. This is an exact - limit. - - fetcher: - An object implementing FetcherInterface that performs the network IO - operations. - - - A file object is created on disk to store the contents of 'url'. - - - tuf.ssl_commons.exceptions.DownloadLengthMismatchError, if there was a - mismatch of observed vs expected lengths while downloading the file. - - securesystemslib.exceptions.FormatError, if any of the arguments are - improperly formatted. - - Any other unforeseen runtime exception. - - - A file object that points to the contents of 'url'. - """ - - # Do all of the arguments have the appropriate format? - # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. - securesystemslib.formats.URL_SCHEMA.check_match(url) - tuf.formats.LENGTH_SCHEMA.check_match(required_length) - - return _download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True) - - - - - -def unsafe_download(url, required_length, fetcher): - """ - - Given the 'url' and 'required_length' of the desired file, open a connection - to 'url', download it, and return the contents of the file. Also ensure - the length of the downloaded file is up to 'required_length', and no larger. - tuf.download.safe_download() may be called if an exact download limit is - preferred. - - - url: - A URL string that represents the location of the file. - - required_length: - An integer value representing the length of the file. This is an upper - limit. - - fetcher: - An object implementing FetcherInterface that performs the network IO - operations. - - - A file object is created on disk to store the contents of 'url'. - - - tuf.ssl_commons.exceptions.DownloadLengthMismatchError, if there was a - mismatch of observed vs expected lengths while downloading the file. - - securesystemslib.exceptions.FormatError, if any of the arguments are - improperly formatted. - - Any other unforeseen runtime exception. - - - A file object that points to the contents of 'url'. - """ - - # Do all of the arguments have the appropriate format? - # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. - securesystemslib.formats.URL_SCHEMA.check_match(url) - tuf.formats.LENGTH_SCHEMA.check_match(required_length) - - return _download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=False) - - - - - -def _download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True): +def download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True): """ Given the url and length of the desired file, this function opens a @@ -180,6 +84,11 @@ def _download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True): A file object that points to the contents of 'url'. """ + # Do all of the arguments have the appropriate format? + # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. + securesystemslib.formats.URL_SCHEMA.check_match(url) + tuf.formats.LENGTH_SCHEMA.check_match(required_length) + # 'url.replace('\\', '/')' is needed for compatibility with Windows-based # systems, because they might use back-slashes in place of forward-slashes. # This converts it to the common format. unquote() replaces %xx escapes in a diff --git a/tuf/client_rework/mirrors.py b/tuf/client_rework/mirrors.py index a9e4dd266b..2b7682645f 100644 --- a/tuf/client_rework/mirrors.py +++ b/tuf/client_rework/mirrors.py @@ -148,10 +148,11 @@ def _mirror_meta_download(filename: str, upper_length: int, file_mirror_errors = {} for file_mirror in file_mirrors: try: - temp_obj = download.unsafe_download( + temp_obj = download.download_file( file_mirror, upper_length, - fetcher) + fetcher, + STRICT_REQUIRED_LENGTH=False) temp_obj.seek(0) yield temp_obj @@ -177,7 +178,7 @@ def _mirror_target_download(fileinfo: str, mirrors_config: Dict, file_mirror_errors = {} for file_mirror in file_mirrors: try: - temp_obj = download.safe_download( + temp_obj = download.download_file( file_mirror, fileinfo['fileinfo']['length'], fetcher) From 3f89c018b3dac0dd71d7f8dae8834df66bfde49d Mon Sep 17 00:00:00 2001 From: Teodora Sechkova Date: Thu, 8 Apr 2021 12:43:37 +0300 Subject: [PATCH 03/11] Reformat client code Run black and isort over the old modules which were moved inside the client directory: - download.py - fetcher.py - mirrors.py - requests_fetcher.py Signed-off-by: Teodora Sechkova --- tuf/client_rework/download.py | 402 ++++++++++++++------------ tuf/client_rework/fetcher.py | 43 +-- tuf/client_rework/mirrors.py | 319 ++++++++++---------- tuf/client_rework/requests_fetcher.py | 299 +++++++++---------- 4 files changed, 556 insertions(+), 507 deletions(-) diff --git a/tuf/client_rework/download.py b/tuf/client_rework/download.py index 4cc855f372..be57224e4c 100644 --- a/tuf/client_rework/download.py +++ b/tuf/client_rework/download.py @@ -23,17 +23,20 @@ metadata of that file. """ + # Help with Python 3 compatibility, where the print statement is a function, an # implicit relative import is invalid, and the '/' operator performs true # division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import logging -import timeit import tempfile +import timeit import securesystemslib import securesystemslib.util @@ -48,185 +51,216 @@ def download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True): - """ - - Given the url and length of the desired file, this function opens a - connection to 'url' and downloads the file while ensuring its length - matches 'required_length' if 'STRICT_REQUIRED_LENGH' is True (If False, - the file's length is not checked and a slow retrieval exception is raised - if the downloaded rate falls below the acceptable rate). - - - url: - A URL string that represents the location of the file. - - required_length: - An integer value representing the length of the file. - - STRICT_REQUIRED_LENGTH: - A Boolean indicator used to signal whether we should perform strict - checking of required_length. True by default. We explicitly set this to - False when we know that we want to turn this off for downloading the - timestamp metadata, which has no signed required_length. - - - A file object is created on disk to store the contents of 'url'. - - - tuf.exceptions.DownloadLengthMismatchError, if there was a - mismatch of observed vs expected lengths while downloading the file. - - securesystemslib.exceptions.FormatError, if any of the arguments are - improperly formatted. - - Any other unforeseen runtime exception. - - - A file object that points to the contents of 'url'. - """ - # Do all of the arguments have the appropriate format? - # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. - securesystemslib.formats.URL_SCHEMA.check_match(url) - tuf.formats.LENGTH_SCHEMA.check_match(required_length) - - # 'url.replace('\\', '/')' is needed for compatibility with Windows-based - # systems, because they might use back-slashes in place of forward-slashes. - # This converts it to the common format. unquote() replaces %xx escapes in a - # url with their single-character equivalent. A back-slash may be encoded as - # %5c in the url, which should also be replaced with a forward slash. - url = six.moves.urllib.parse.unquote(url).replace('\\', '/') - logger.info('Downloading: ' + repr(url)) - - # This is the temporary file that we will return to contain the contents of - # the downloaded file. - temp_file = tempfile.TemporaryFile() - - average_download_speed = 0 - number_of_bytes_received = 0 - - try: - chunks = fetcher.fetch(url, required_length) - start_time = timeit.default_timer() - for chunk in chunks: - - stop_time = timeit.default_timer() - temp_file.write(chunk) - - # Measure the average download speed. - number_of_bytes_received += len(chunk) - seconds_spent_receiving = stop_time - start_time - average_download_speed = number_of_bytes_received / seconds_spent_receiving - - if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: - logger.debug('The average download speed dropped below the minimum' - ' average download speed set in tuf.settings.py. Stopping the' - ' download!') - break - - else: - logger.debug('The average download speed has not dipped below the' - ' minimum average download speed set in tuf.settings.py.') - - # Does the total number of downloaded bytes match the required length? - _check_downloaded_length(number_of_bytes_received, required_length, - STRICT_REQUIRED_LENGTH=STRICT_REQUIRED_LENGTH, - average_download_speed=average_download_speed) - - except Exception: - # Close 'temp_file'. Any written data is lost. - temp_file.close() - logger.debug('Could not download URL: ' + repr(url)) - raise - - else: - return temp_file - - - - -def _check_downloaded_length(total_downloaded, required_length, - STRICT_REQUIRED_LENGTH=True, - average_download_speed=None): - """ - - A helper function which checks whether the total number of downloaded bytes - matches our expectation. - - - total_downloaded: - The total number of bytes supposedly downloaded for the file in question. - - required_length: - The total number of bytes expected of the file as seen from its metadata. - The Timestamp role is always downloaded without a known file length, and - the Root role when the client cannot download any of the required - top-level roles. In both cases, 'required_length' is actually an upper - limit on the length of the downloaded file. - - STRICT_REQUIRED_LENGTH: - A Boolean indicator used to signal whether we should perform strict - checking of required_length. True by default. We explicitly set this to - False when we know that we want to turn this off for downloading the - timestamp metadata, which has no signed required_length. - - average_download_speed: - The average download speed for the downloaded file. - - - None. - - - securesystemslib.exceptions.DownloadLengthMismatchError, if - STRICT_REQUIRED_LENGTH is True and total_downloaded is not equal - required_length. - - tuf.exceptions.SlowRetrievalError, if the total downloaded was - done in less than the acceptable download speed (as set in - tuf.settings.py). - - - None. - """ - - if total_downloaded == required_length: - logger.info('Downloaded ' + str(total_downloaded) + ' bytes out of the' - ' expected ' + str(required_length) + ' bytes.') - - else: - difference_in_bytes = abs(total_downloaded - required_length) - - # What we downloaded is not equal to the required length, but did we ask - # for strict checking of required length? - if STRICT_REQUIRED_LENGTH: - logger.info('Downloaded ' + str(total_downloaded) + ' bytes, but' - ' expected ' + str(required_length) + ' bytes. There is a difference' - ' of ' + str(difference_in_bytes) + ' bytes.') + """ + + Given the url and length of the desired file, this function opens a + connection to 'url' and downloads the file while ensuring its length + matches 'required_length' if 'STRICT_REQUIRED_LENGH' is True (If False, + the file's length is not checked and a slow retrieval exception is raised + if the downloaded rate falls below the acceptable rate). + + + url: + A URL string that represents the location of the file. + + required_length: + An integer value representing the length of the file. + + STRICT_REQUIRED_LENGTH: + A Boolean indicator used to signal whether we should perform strict + checking of required_length. True by default. We explicitly set this to + False when we know that we want to turn this off for downloading the + timestamp metadata, which has no signed required_length. + + + A file object is created on disk to store the contents of 'url'. + + + tuf.exceptions.DownloadLengthMismatchError, if there was a + mismatch of observed vs expected lengths while downloading the file. + + securesystemslib.exceptions.FormatError, if any of the arguments are + improperly formatted. + + Any other unforeseen runtime exception. + + + A file object that points to the contents of 'url'. + """ + # Do all of the arguments have the appropriate format? + # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. + securesystemslib.formats.URL_SCHEMA.check_match(url) + tuf.formats.LENGTH_SCHEMA.check_match(required_length) + + # 'url.replace('\\', '/')' is needed for compatibility with Windows-based + # systems, because they might use back-slashes in place of forward-slashes. + # This converts it to the common format. unquote() replaces %xx escapes in a + # url with their single-character equivalent. A back-slash may be encoded as + # %5c in the url, which should also be replaced with a forward slash. + url = six.moves.urllib.parse.unquote(url).replace("\\", "/") + logger.info("Downloading: " + repr(url)) + + # This is the temporary file that we will return to contain the contents of + # the downloaded file. + temp_file = tempfile.TemporaryFile() + + average_download_speed = 0 + number_of_bytes_received = 0 + + try: + chunks = fetcher.fetch(url, required_length) + start_time = timeit.default_timer() + for chunk in chunks: + + stop_time = timeit.default_timer() + temp_file.write(chunk) + + # Measure the average download speed. + number_of_bytes_received += len(chunk) + seconds_spent_receiving = stop_time - start_time + average_download_speed = ( + number_of_bytes_received / seconds_spent_receiving + ) + + if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: + logger.debug( + "The average download speed dropped below the minimum" + " average download speed set in tuf.settings.py. Stopping the" + " download!" + ) + break + + else: + logger.debug( + "The average download speed has not dipped below the" + " minimum average download speed set in tuf.settings.py." + ) + + # Does the total number of downloaded bytes match the required length? + _check_downloaded_length( + number_of_bytes_received, + required_length, + STRICT_REQUIRED_LENGTH=STRICT_REQUIRED_LENGTH, + average_download_speed=average_download_speed, + ) + + except Exception: + # Close 'temp_file'. Any written data is lost. + temp_file.close() + logger.debug("Could not download URL: " + repr(url)) + raise - # If the average download speed is below a certain threshold, we flag - # this as a possible slow-retrieval attack. - logger.debug('Average download speed: ' + repr(average_download_speed)) - logger.debug('Minimum average download speed: ' + repr(tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED)) - - if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: - raise tuf.exceptions.SlowRetrievalError(average_download_speed) - - else: - logger.debug('Good average download speed: ' + - repr(average_download_speed) + ' bytes per second') - - raise tuf.exceptions.DownloadLengthMismatchError(required_length, total_downloaded) + else: + return temp_file + + +def _check_downloaded_length( + total_downloaded, + required_length, + STRICT_REQUIRED_LENGTH=True, + average_download_speed=None, +): + """ + + A helper function which checks whether the total number of downloaded bytes + matches our expectation. + + + total_downloaded: + The total number of bytes supposedly downloaded for the file in question. + + required_length: + The total number of bytes expected of the file as seen from its metadata. + The Timestamp role is always downloaded without a known file length, and + the Root role when the client cannot download any of the required + top-level roles. In both cases, 'required_length' is actually an upper + limit on the length of the downloaded file. + + STRICT_REQUIRED_LENGTH: + A Boolean indicator used to signal whether we should perform strict + checking of required_length. True by default. We explicitly set this to + False when we know that we want to turn this off for downloading the + timestamp metadata, which has no signed required_length. + + average_download_speed: + The average download speed for the downloaded file. + + + None. + + + securesystemslib.exceptions.DownloadLengthMismatchError, if + STRICT_REQUIRED_LENGTH is True and total_downloaded is not equal + required_length. + + tuf.exceptions.SlowRetrievalError, if the total downloaded was + done in less than the acceptable download speed (as set in + tuf.settings.py). + + + None. + """ + + if total_downloaded == required_length: + logger.info( + "Downloaded " + str(total_downloaded) + " bytes out of the" + " expected " + str(required_length) + " bytes." + ) else: - # We specifically disabled strict checking of required length, but we - # will log a warning anyway. This is useful when we wish to download the - # Timestamp or Root metadata, for which we have no signed metadata; so, - # we must guess a reasonable required_length for it. - if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: - raise tuf.exceptions.SlowRetrievalError(average_download_speed) - - else: - logger.debug('Good average download speed: ' + - repr(average_download_speed) + ' bytes per second') - - logger.info('Downloaded ' + str(total_downloaded) + ' bytes out of an' - ' upper limit of ' + str(required_length) + ' bytes.') + difference_in_bytes = abs(total_downloaded - required_length) + + # What we downloaded is not equal to the required length, but did we ask + # for strict checking of required length? + if STRICT_REQUIRED_LENGTH: + logger.info( + "Downloaded " + str(total_downloaded) + " bytes, but" + " expected " + + str(required_length) + + " bytes. There is a difference" + " of " + str(difference_in_bytes) + " bytes." + ) + + # If the average download speed is below a certain threshold, we flag + # this as a possible slow-retrieval attack. + logger.debug( + "Average download speed: " + repr(average_download_speed) + ) + logger.debug( + "Minimum average download speed: " + + repr(tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED) + ) + + if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: + raise tuf.exceptions.SlowRetrievalError(average_download_speed) + + else: + logger.debug( + "Good average download speed: " + + repr(average_download_speed) + + " bytes per second" + ) + + raise tuf.exceptions.DownloadLengthMismatchError( + required_length, total_downloaded + ) + + else: + # We specifically disabled strict checking of required length, but we + # will log a warning anyway. This is useful when we wish to download the + # Timestamp or Root metadata, for which we have no signed metadata; so, + # we must guess a reasonable required_length for it. + if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: + raise tuf.exceptions.SlowRetrievalError(average_download_speed) + + else: + logger.debug( + "Good average download speed: " + + repr(average_download_speed) + + " bytes per second" + ) + + logger.info( + "Downloaded " + str(total_downloaded) + " bytes out of an" + " upper limit of " + str(required_length) + " bytes." + ) diff --git a/tuf/client_rework/fetcher.py b/tuf/client_rework/fetcher.py index 8768bdd4b9..2b6de6f837 100644 --- a/tuf/client_rework/fetcher.py +++ b/tuf/client_rework/fetcher.py @@ -7,32 +7,33 @@ # Imports import abc + # Classes -class FetcherInterface(): - """Defines an interface for abstract network download. +class FetcherInterface: + """Defines an interface for abstract network download. - By providing a concrete implementation of the abstract interface, - users of the framework can plug-in their preferred/customized - network stack. - """ + By providing a concrete implementation of the abstract interface, + users of the framework can plug-in their preferred/customized + network stack. + """ - __metaclass__ = abc.ABCMeta + __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def fetch(self, url, required_length): - """Fetches the contents of HTTP/HTTPS url from a remote server. + @abc.abstractmethod + def fetch(self, url, required_length): + """Fetches the contents of HTTP/HTTPS url from a remote server. - Ensures the length of the downloaded data is up to 'required_length'. + Ensures the length of the downloaded data is up to 'required_length'. - Arguments: - url: A URL string that represents a file location. - required_length: An integer value representing the file length in bytes. + Arguments: + url: A URL string that represents a file location. + required_length: An integer value representing the file length in bytes. - Raises: - tuf.exceptions.SlowRetrievalError: A timeout occurs while receiving data. - tuf.exceptions.FetcherHTTPError: An HTTP error code is received. + Raises: + tuf.exceptions.SlowRetrievalError: A timeout occurs while receiving data. + tuf.exceptions.FetcherHTTPError: An HTTP error code is received. - Returns: - A bytes iterator - """ - raise NotImplementedError # pragma: no cover + Returns: + A bytes iterator + """ + raise NotImplementedError # pragma: no cover diff --git a/tuf/client_rework/mirrors.py b/tuf/client_rework/mirrors.py index 2b7682645f..afcbdb9b0f 100644 --- a/tuf/client_rework/mirrors.py +++ b/tuf/client_rework/mirrors.py @@ -22,174 +22,179 @@ of the file with respect to the base url. """ + # Help with Python 3 compatibility, where the print statement is a function, an # implicit relative import is invalid, and the '/' operator performs true # division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - -from typing import TextIO, BinaryIO, Dict +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import os - -import tuf -import tuf.formats -import tuf.client_rework.download as download +from typing import BinaryIO, Dict, TextIO import securesystemslib import six +import tuf +import tuf.client_rework.download as download +import tuf.formats + # The type of file to be downloaded from a repository. The # 'get_list_of_mirrors' function supports these file types. -_SUPPORTED_FILE_TYPES = ['meta', 'target'] +_SUPPORTED_FILE_TYPES = ["meta", "target"] def get_list_of_mirrors(file_type, file_path, mirrors_dict): - """ - - Get a list of mirror urls from a mirrors dictionary, provided the type - and the path of the file with respect to the base url. - - - file_type: - Type of data needed for download, must correspond to one of the strings - in the list ['meta', 'target']. 'meta' for metadata file type or - 'target' for target file type. It should correspond to - NAME_SCHEMA format. - - file_path: - A relative path to the file that corresponds to RELPATH_SCHEMA format. - Ex: 'http://url_prefix/targets_path/file_path' - - mirrors_dict: - A mirrors_dict object that corresponds to MIRRORDICT_SCHEMA, where - keys are strings and values are MIRROR_SCHEMA. An example format - of MIRROR_SCHEMA: - - {'url_prefix': 'http://localhost:8001', - 'metadata_path': 'metadata/', - 'targets_path': 'targets/', - 'confined_target_dirs': ['targets/snapshot1/', ...], - 'custom': {...}} - - The 'custom' field is optional. - - - securesystemslib.exceptions.Error, on unsupported 'file_type'. - - securesystemslib.exceptions.FormatError, on bad argument. - - - List of mirror urls corresponding to the file_type and file_path. If no - match is found, empty list is returned. - """ - - # Checking if all the arguments have appropriate format. - tuf.formats.RELPATH_SCHEMA.check_match(file_path) - tuf.formats.MIRRORDICT_SCHEMA.check_match(mirrors_dict) - securesystemslib.formats.NAME_SCHEMA.check_match(file_type) - - # Verify 'file_type' is supported. - if file_type not in _SUPPORTED_FILE_TYPES: - raise securesystemslib.exceptions.Error('Invalid file_type argument.' - ' Supported file types: ' + repr(_SUPPORTED_FILE_TYPES)) - path_key = 'metadata_path' if file_type == 'meta' else 'targets_path' - - # Reference to 'securesystemslib.util.file_in_confined_directories()' (improve - # readability). This function checks whether a mirror should serve a file to - # the client. A client may be confined to certain paths on a repository - # mirror when fetching target files. This field may be set by the client - # when the repository mirror is added to the 'tuf.client.updater.Updater' - # object. - in_confined_directory = securesystemslib.util.file_in_confined_directories - - list_of_mirrors = [] - for junk, mirror_info in six.iteritems(mirrors_dict): - # Does mirror serve this file type at all? - path = mirror_info.get(path_key) - if path is None: - continue - - # for targets, ensure directory confinement - if path_key == 'targets_path': - full_filepath = os.path.join(path, file_path) - confined_target_dirs = mirror_info.get('confined_target_dirs') - # confined_target_dirs is an optional field - if confined_target_dirs and not in_confined_directory(full_filepath, - confined_target_dirs): - continue - - # urllib.quote(string) replaces special characters in string using the %xx - # escape. This is done to avoid parsing issues of the URL on the server - # side. Do *NOT* pass URLs with Unicode characters without first encoding - # the URL as UTF-8. We need a long-term solution with #61. - # http://bugs.python.org/issue1712522 - file_path = six.moves.urllib.parse.quote(file_path) - url = os.path.join(mirror_info['url_prefix'], path, file_path) - - # The above os.path.join() result as well as input file_path may be - # invalid on windows (might contain both separator types), see #1077. - # Make sure the URL doesn't contain backward slashes on Windows. - list_of_mirrors.append(url.replace('\\', '/')) - - return list_of_mirrors - - -def _mirror_meta_download(filename: str, upper_length: int, - mirrors_config: Dict, - fetcher: "FetcherInterface") -> TextIO: - """ - Download metadata file from the list of metadata mirrors - """ - file_mirrors = get_list_of_mirrors('meta', filename, mirrors_config) - - file_mirror_errors = {} - for file_mirror in file_mirrors: - try: - temp_obj = download.download_file( - file_mirror, - upper_length, - fetcher, - STRICT_REQUIRED_LENGTH=False) - - temp_obj.seek(0) - yield temp_obj - - except Exception as exception: - file_mirror_errors[file_mirror] = exception - - finally: - if file_mirror_errors: - raise tuf.exceptions.NoWorkingMirrorError( - file_mirror_errors) - - -def _mirror_target_download(fileinfo: str, mirrors_config: Dict, - fetcher: "FetcherInterface") -> BinaryIO: - """ - Download target file from the list of target mirrors - """ - # full_filename = _get_full_name(filename) - file_mirrors = get_list_of_mirrors('target', fileinfo['filepath'], - mirrors_config) - - file_mirror_errors = {} - for file_mirror in file_mirrors: - try: - temp_obj = download.download_file( - file_mirror, - fileinfo['fileinfo']['length'], - fetcher) - - temp_obj.seek(0) - yield temp_obj - - except Exception as exception: - file_mirror_errors[file_mirror] = exception - - finally: - if file_mirror_errors: - raise tuf.exceptions.NoWorkingMirrorError( - file_mirror_errors) + """ + + Get a list of mirror urls from a mirrors dictionary, provided the type + and the path of the file with respect to the base url. + + + file_type: + Type of data needed for download, must correspond to one of the strings + in the list ['meta', 'target']. 'meta' for metadata file type or + 'target' for target file type. It should correspond to + NAME_SCHEMA format. + + file_path: + A relative path to the file that corresponds to RELPATH_SCHEMA format. + Ex: 'http://url_prefix/targets_path/file_path' + + mirrors_dict: + A mirrors_dict object that corresponds to MIRRORDICT_SCHEMA, where + keys are strings and values are MIRROR_SCHEMA. An example format + of MIRROR_SCHEMA: + + {'url_prefix': 'http://localhost:8001', + 'metadata_path': 'metadata/', + 'targets_path': 'targets/', + 'confined_target_dirs': ['targets/snapshot1/', ...], + 'custom': {...}} + + The 'custom' field is optional. + + + securesystemslib.exceptions.Error, on unsupported 'file_type'. + + securesystemslib.exceptions.FormatError, on bad argument. + + + List of mirror urls corresponding to the file_type and file_path. If no + match is found, empty list is returned. + """ + + # Checking if all the arguments have appropriate format. + tuf.formats.RELPATH_SCHEMA.check_match(file_path) + tuf.formats.MIRRORDICT_SCHEMA.check_match(mirrors_dict) + securesystemslib.formats.NAME_SCHEMA.check_match(file_type) + + # Verify 'file_type' is supported. + if file_type not in _SUPPORTED_FILE_TYPES: + raise securesystemslib.exceptions.Error( + "Invalid file_type argument." + " Supported file types: " + repr(_SUPPORTED_FILE_TYPES) + ) + path_key = "metadata_path" if file_type == "meta" else "targets_path" + + # Reference to 'securesystemslib.util.file_in_confined_directories()' (improve + # readability). This function checks whether a mirror should serve a file to + # the client. A client may be confined to certain paths on a repository + # mirror when fetching target files. This field may be set by the client + # when the repository mirror is added to the 'tuf.client.updater.Updater' + # object. + in_confined_directory = securesystemslib.util.file_in_confined_directories + + list_of_mirrors = [] + for junk, mirror_info in six.iteritems(mirrors_dict): + # Does mirror serve this file type at all? + path = mirror_info.get(path_key) + if path is None: + continue + + # for targets, ensure directory confinement + if path_key == "targets_path": + full_filepath = os.path.join(path, file_path) + confined_target_dirs = mirror_info.get("confined_target_dirs") + # confined_target_dirs is an optional field + if confined_target_dirs and not in_confined_directory( + full_filepath, confined_target_dirs + ): + continue + + # urllib.quote(string) replaces special characters in string using the %xx + # escape. This is done to avoid parsing issues of the URL on the server + # side. Do *NOT* pass URLs with Unicode characters without first encoding + # the URL as UTF-8. We need a long-term solution with #61. + # http://bugs.python.org/issue1712522 + file_path = six.moves.urllib.parse.quote(file_path) + url = os.path.join(mirror_info["url_prefix"], path, file_path) + + # The above os.path.join() result as well as input file_path may be + # invalid on windows (might contain both separator types), see #1077. + # Make sure the URL doesn't contain backward slashes on Windows. + list_of_mirrors.append(url.replace("\\", "/")) + + return list_of_mirrors + + +def _mirror_meta_download( + filename: str, + upper_length: int, + mirrors_config: Dict, + fetcher: "FetcherInterface", +) -> TextIO: + """ + Download metadata file from the list of metadata mirrors + """ + file_mirrors = get_list_of_mirrors("meta", filename, mirrors_config) + + file_mirror_errors = {} + for file_mirror in file_mirrors: + try: + temp_obj = download.download_file( + file_mirror, upper_length, fetcher, STRICT_REQUIRED_LENGTH=False + ) + + temp_obj.seek(0) + yield temp_obj + + except Exception as exception: + file_mirror_errors[file_mirror] = exception + + finally: + if file_mirror_errors: + raise tuf.exceptions.NoWorkingMirrorError(file_mirror_errors) + + +def _mirror_target_download( + fileinfo: str, mirrors_config: Dict, fetcher: "FetcherInterface" +) -> BinaryIO: + """ + Download target file from the list of target mirrors + """ + # full_filename = _get_full_name(filename) + file_mirrors = get_list_of_mirrors( + "target", fileinfo["filepath"], mirrors_config + ) + + file_mirror_errors = {} + for file_mirror in file_mirrors: + try: + temp_obj = download.download_file( + file_mirror, fileinfo["fileinfo"]["length"], fetcher + ) + + temp_obj.seek(0) + yield temp_obj + + except Exception as exception: + file_mirror_errors[file_mirror] = exception + + finally: + if file_mirror_errors: + raise tuf.exceptions.NoWorkingMirrorError(file_mirror_errors) diff --git a/tuf/client_rework/requests_fetcher.py b/tuf/client_rework/requests_fetcher.py index 8074890d25..6f5e89ec4e 100644 --- a/tuf/client_rework/requests_fetcher.py +++ b/tuf/client_rework/requests_fetcher.py @@ -5,17 +5,16 @@ library. """ -# Imports -import requests -import six import logging import time +# Imports +import requests +import six import urllib3.exceptions import tuf.exceptions import tuf.settings - from tuf.client_rework.fetcher import FetcherInterface # Globals @@ -23,151 +22,161 @@ # Classess class RequestsFetcher(FetcherInterface): - """A concrete implementation of FetcherInterface based on the Requests + """A concrete implementation of FetcherInterface based on the Requests library. Attributes: _sessions: A dictionary of Requests.Session objects storing a separate session per scheme+hostname combination. - """ - - def __init__(self): - # From http://docs.python-requests.org/en/master/user/advanced/#session-objects: - # - # "The Session object allows you to persist certain parameters across - # requests. It also persists cookies across all requests made from the - # Session instance, and will use urllib3's connection pooling. So if you're - # making several requests to the same host, the underlying TCP connection - # will be reused, which can result in a significant performance increase - # (see HTTP persistent connection)." - # - # NOTE: We use a separate requests.Session per scheme+hostname combination, - # in order to reuse connections to the same hostname to improve efficiency, - # but avoiding sharing state between different hosts-scheme combinations to - # minimize subtle security issues. Some cookies may not be HTTP-safe. - self._sessions = {} - - - def fetch(self, url, required_length): - """Fetches the contents of HTTP/HTTPS url from a remote server. - - Ensures the length of the downloaded data is up to 'required_length'. - - Arguments: - url: A URL string that represents a file location. - required_length: An integer value representing the file length in bytes. - - Raises: - tuf.exceptions.SlowRetrievalError: A timeout occurs while receiving data. - tuf.exceptions.FetcherHTTPError: An HTTP error code is received. - - Returns: - A bytes iterator - """ - # Get a customized session for each new schema+hostname combination. - session = self._get_session(url) - - # Get the requests.Response object for this URL. - # - # Defer downloading the response body with stream=True. - # Always set the timeout. This timeout value is interpreted by requests as: - # - connect timeout (max delay before first byte is received) - # - read (gap) timeout (max delay between bytes received) - response = session.get(url, stream=True, - timeout=tuf.settings.SOCKET_TIMEOUT) - # Check response status. - try: - response.raise_for_status() - except requests.HTTPError as e: - response.close() - status = e.response.status_code - raise tuf.exceptions.FetcherHTTPError(str(e), status) - - - # Define a generator function to be returned by fetch. This way the caller - # of fetch can differentiate between connection and actual data download - # and measure download times accordingly. - def chunks(): - try: - bytes_received = 0 - while True: - # We download a fixed chunk of data in every round. This is so that we - # can defend against slow retrieval attacks. Furthermore, we do not - # wish to download an extremely large file in one shot. - # Before beginning the round, sleep (if set) for a short amount of - # time so that the CPU is not hogged in the while loop. - if tuf.settings.SLEEP_BEFORE_ROUND: - time.sleep(tuf.settings.SLEEP_BEFORE_ROUND) - - read_amount = min( - tuf.settings.CHUNK_SIZE, required_length - bytes_received) - - # NOTE: This may not handle some servers adding a Content-Encoding - # header, which may cause urllib3 to misbehave: - # https://github.com/pypa/pip/blob/404838abcca467648180b358598c597b74d568c9/src/pip/_internal/download.py#L547-L582 - data = response.raw.read(read_amount) - bytes_received += len(data) - - # We might have no more data to read. Check number of bytes downloaded. - if not data: - logger.debug('Downloaded ' + repr(bytes_received) + '/' + - repr(required_length) + ' bytes.') - - # Finally, we signal that the download is complete. - break - - yield data - - if bytes_received >= required_length: - break - - except urllib3.exceptions.ReadTimeoutError as e: - raise tuf.exceptions.SlowRetrievalError(str(e)) - - finally: - response.close() - - return chunks() - - - - def _get_session(self, url): - """Returns a different customized requests.Session per schema+hostname - combination. """ - # Use a different requests.Session per schema+hostname combination, to - # reuse connections while minimizing subtle security issues. - parsed_url = six.moves.urllib.parse.urlparse(url) - - if not parsed_url.scheme or not parsed_url.hostname: - raise tuf.exceptions.URLParsingError( - 'Could not get scheme and hostname from URL: ' + url) - - session_index = parsed_url.scheme + '+' + parsed_url.hostname - - logger.debug('url: ' + url) - logger.debug('session index: ' + session_index) - - session = self._sessions.get(session_index) - - if not session: - session = requests.Session() - self._sessions[session_index] = session - - # Attach some default headers to every Session. - requests_user_agent = session.headers['User-Agent'] - # Follows the RFC: https://tools.ietf.org/html/rfc7231#section-5.5.3 - tuf_user_agent = 'tuf/' + tuf.__version__ + ' ' + requests_user_agent - session.headers.update({ - # Tell the server not to compress or modify anything. - # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding#Directives - 'Accept-Encoding': 'identity', - # The TUF user agent. - 'User-Agent': tuf_user_agent}) - - logger.debug('Made new session for ' + session_index) - - else: - logger.debug('Reusing session for ' + session_index) - return session + def __init__(self): + # From http://docs.python-requests.org/en/master/user/advanced/#session-objects: + # + # "The Session object allows you to persist certain parameters across + # requests. It also persists cookies across all requests made from the + # Session instance, and will use urllib3's connection pooling. So if you're + # making several requests to the same host, the underlying TCP connection + # will be reused, which can result in a significant performance increase + # (see HTTP persistent connection)." + # + # NOTE: We use a separate requests.Session per scheme+hostname combination, + # in order to reuse connections to the same hostname to improve efficiency, + # but avoiding sharing state between different hosts-scheme combinations to + # minimize subtle security issues. Some cookies may not be HTTP-safe. + self._sessions = {} + + def fetch(self, url, required_length): + """Fetches the contents of HTTP/HTTPS url from a remote server. + + Ensures the length of the downloaded data is up to 'required_length'. + + Arguments: + url: A URL string that represents a file location. + required_length: An integer value representing the file length in bytes. + + Raises: + tuf.exceptions.SlowRetrievalError: A timeout occurs while receiving data. + tuf.exceptions.FetcherHTTPError: An HTTP error code is received. + + Returns: + A bytes iterator + """ + # Get a customized session for each new schema+hostname combination. + session = self._get_session(url) + + # Get the requests.Response object for this URL. + # + # Defer downloading the response body with stream=True. + # Always set the timeout. This timeout value is interpreted by requests as: + # - connect timeout (max delay before first byte is received) + # - read (gap) timeout (max delay between bytes received) + response = session.get( + url, stream=True, timeout=tuf.settings.SOCKET_TIMEOUT + ) + # Check response status. + try: + response.raise_for_status() + except requests.HTTPError as e: + response.close() + status = e.response.status_code + raise tuf.exceptions.FetcherHTTPError(str(e), status) + + # Define a generator function to be returned by fetch. This way the caller + # of fetch can differentiate between connection and actual data download + # and measure download times accordingly. + def chunks(): + try: + bytes_received = 0 + while True: + # We download a fixed chunk of data in every round. This is so that we + # can defend against slow retrieval attacks. Furthermore, we do not + # wish to download an extremely large file in one shot. + # Before beginning the round, sleep (if set) for a short amount of + # time so that the CPU is not hogged in the while loop. + if tuf.settings.SLEEP_BEFORE_ROUND: + time.sleep(tuf.settings.SLEEP_BEFORE_ROUND) + + read_amount = min( + tuf.settings.CHUNK_SIZE, + required_length - bytes_received, + ) + + # NOTE: This may not handle some servers adding a Content-Encoding + # header, which may cause urllib3 to misbehave: + # https://github.com/pypa/pip/blob/404838abcca467648180b358598c597b74d568c9/src/pip/_internal/download.py#L547-L582 + data = response.raw.read(read_amount) + bytes_received += len(data) + + # We might have no more data to read. Check number of bytes downloaded. + if not data: + logger.debug( + "Downloaded " + + repr(bytes_received) + + "/" + + repr(required_length) + + " bytes." + ) + + # Finally, we signal that the download is complete. + break + + yield data + + if bytes_received >= required_length: + break + + except urllib3.exceptions.ReadTimeoutError as e: + raise tuf.exceptions.SlowRetrievalError(str(e)) + + finally: + response.close() + + return chunks() + + def _get_session(self, url): + """Returns a different customized requests.Session per schema+hostname + combination. + """ + # Use a different requests.Session per schema+hostname combination, to + # reuse connections while minimizing subtle security issues. + parsed_url = six.moves.urllib.parse.urlparse(url) + + if not parsed_url.scheme or not parsed_url.hostname: + raise tuf.exceptions.URLParsingError( + "Could not get scheme and hostname from URL: " + url + ) + + session_index = parsed_url.scheme + "+" + parsed_url.hostname + + logger.debug("url: " + url) + logger.debug("session index: " + session_index) + + session = self._sessions.get(session_index) + + if not session: + session = requests.Session() + self._sessions[session_index] = session + + # Attach some default headers to every Session. + requests_user_agent = session.headers["User-Agent"] + # Follows the RFC: https://tools.ietf.org/html/rfc7231#section-5.5.3 + tuf_user_agent = ( + "tuf/" + tuf.__version__ + " " + requests_user_agent + ) + session.headers.update( + { + # Tell the server not to compress or modify anything. + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding#Directives + "Accept-Encoding": "identity", + # The TUF user agent. + "User-Agent": tuf_user_agent, + } + ) + + logger.debug("Made new session for " + session_index) + + else: + logger.debug("Reusing session for " + session_index) + + return session From 1c6ab320cdcd4038c686f0d5bdfe6984a1af220d Mon Sep 17 00:00:00 2001 From: Teodora Sechkova Date: Thu, 8 Apr 2021 12:56:29 +0300 Subject: [PATCH 04/11] Fix pylint C0301: Line too long Fix line lenght exceeding 80 characters since the black formatter does not wrap lines inside comments. Signed-off-by: Teodora Sechkova --- tuf/client_rework/download.py | 33 +++++++------- tuf/client_rework/fetcher.py | 12 ++--- tuf/client_rework/mirrors.py | 20 ++++----- tuf/client_rework/requests_fetcher.py | 63 +++++++++++++++------------ 4 files changed, 70 insertions(+), 58 deletions(-) diff --git a/tuf/client_rework/download.py b/tuf/client_rework/download.py index be57224e4c..d04cc2baa3 100644 --- a/tuf/client_rework/download.py +++ b/tuf/client_rework/download.py @@ -94,9 +94,10 @@ def download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True): # 'url.replace('\\', '/')' is needed for compatibility with Windows-based # systems, because they might use back-slashes in place of forward-slashes. - # This converts it to the common format. unquote() replaces %xx escapes in a - # url with their single-character equivalent. A back-slash may be encoded as - # %5c in the url, which should also be replaced with a forward slash. + # This converts it to the common format. unquote() replaces %xx escapes in + # a url with their single-character equivalent. A back-slash may be + # encoded as %5c in the url, which should also be replaced with a forward + # slash. url = six.moves.urllib.parse.unquote(url).replace("\\", "/") logger.info("Downloading: " + repr(url)) @@ -125,8 +126,8 @@ def download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True): if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: logger.debug( "The average download speed dropped below the minimum" - " average download speed set in tuf.settings.py. Stopping the" - " download!" + " average download speed set in tuf.settings.py." + " Stopping the download!" ) break @@ -162,15 +163,16 @@ def _check_downloaded_length( ): """ - A helper function which checks whether the total number of downloaded bytes - matches our expectation. + A helper function which checks whether the total number of downloaded + bytes matches our expectation. total_downloaded: - The total number of bytes supposedly downloaded for the file in question. + The total number of bytes supposedly downloaded for the file in + question. required_length: - The total number of bytes expected of the file as seen from its metadata. + The total number of bytes expected of the file as seen from its metadata The Timestamp role is always downloaded without a known file length, and the Root role when the client cannot download any of the required top-level roles. In both cases, 'required_length' is actually an upper @@ -221,8 +223,8 @@ def _check_downloaded_length( " of " + str(difference_in_bytes) + " bytes." ) - # If the average download speed is below a certain threshold, we flag - # this as a possible slow-retrieval attack. + # If the average download speed is below a certain threshold, we + # flag this as a possible slow-retrieval attack. logger.debug( "Average download speed: " + repr(average_download_speed) ) @@ -246,10 +248,11 @@ def _check_downloaded_length( ) else: - # We specifically disabled strict checking of required length, but we - # will log a warning anyway. This is useful when we wish to download the - # Timestamp or Root metadata, for which we have no signed metadata; so, - # we must guess a reasonable required_length for it. + # We specifically disabled strict checking of required length, but + # we will log a warning anyway. This is useful when we wish to + # download the Timestamp or Root metadata, for which we have no + # signed metadata; so, we must guess a reasonable required_length + # for it. if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: raise tuf.exceptions.SlowRetrievalError(average_download_speed) diff --git a/tuf/client_rework/fetcher.py b/tuf/client_rework/fetcher.py index 2b6de6f837..8a6cae34d7 100644 --- a/tuf/client_rework/fetcher.py +++ b/tuf/client_rework/fetcher.py @@ -26,14 +26,16 @@ def fetch(self, url, required_length): Ensures the length of the downloaded data is up to 'required_length'. Arguments: - url: A URL string that represents a file location. - required_length: An integer value representing the file length in bytes. + url: A URL string that represents a file location. + required_length: An integer value representing the file length in + bytes. Raises: - tuf.exceptions.SlowRetrievalError: A timeout occurs while receiving data. - tuf.exceptions.FetcherHTTPError: An HTTP error code is received. + tuf.exceptions.SlowRetrievalError: A timeout occurs while receiving + data. + tuf.exceptions.FetcherHTTPError: An HTTP error code is received. Returns: - A bytes iterator + A bytes iterator """ raise NotImplementedError # pragma: no cover diff --git a/tuf/client_rework/mirrors.py b/tuf/client_rework/mirrors.py index afcbdb9b0f..0bdf07e2e6 100644 --- a/tuf/client_rework/mirrors.py +++ b/tuf/client_rework/mirrors.py @@ -101,12 +101,12 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict): ) path_key = "metadata_path" if file_type == "meta" else "targets_path" - # Reference to 'securesystemslib.util.file_in_confined_directories()' (improve - # readability). This function checks whether a mirror should serve a file to - # the client. A client may be confined to certain paths on a repository - # mirror when fetching target files. This field may be set by the client - # when the repository mirror is added to the 'tuf.client.updater.Updater' - # object. + # Reference to 'securesystemslib.util.file_in_confined_directories()' + # (improve readability). This function checks whether a mirror should + # serve a file to the client. A client may be confined to certain paths + # on a repository mirror when fetching target files. This field may be set + # by the client when the repository mirror is added to the + # 'tuf.client.updater.Updater' object. in_confined_directory = securesystemslib.util.file_in_confined_directories list_of_mirrors = [] @@ -126,10 +126,10 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict): ): continue - # urllib.quote(string) replaces special characters in string using the %xx - # escape. This is done to avoid parsing issues of the URL on the server - # side. Do *NOT* pass URLs with Unicode characters without first encoding - # the URL as UTF-8. We need a long-term solution with #61. + # urllib.quote(string) replaces special characters in string using + # the %xx escape. This is done to avoid parsing issues of the URL + # on the server side. Do *NOT* pass URLs with Unicode characters without + # first encoding the URL as UTF-8. Needed a long-term solution with #61. # http://bugs.python.org/issue1712522 file_path = six.moves.urllib.parse.quote(file_path) url = os.path.join(mirror_info["url_prefix"], path, file_path) diff --git a/tuf/client_rework/requests_fetcher.py b/tuf/client_rework/requests_fetcher.py index 6f5e89ec4e..545a23feff 100644 --- a/tuf/client_rework/requests_fetcher.py +++ b/tuf/client_rework/requests_fetcher.py @@ -26,24 +26,25 @@ class RequestsFetcher(FetcherInterface): library. Attributes: - _sessions: A dictionary of Requests.Session objects storing a separate - session per scheme+hostname combination. + _sessions: A dictionary of Requests.Session objects storing a separate + session per scheme+hostname combination. """ def __init__(self): - # From http://docs.python-requests.org/en/master/user/advanced/#session-objects: + # http://docs.python-requests.org/en/master/user/advanced/#session-objects: # # "The Session object allows you to persist certain parameters across # requests. It also persists cookies across all requests made from the - # Session instance, and will use urllib3's connection pooling. So if you're - # making several requests to the same host, the underlying TCP connection - # will be reused, which can result in a significant performance increase - # (see HTTP persistent connection)." + # Session instance, and will use urllib3's connection pooling. So if + # you're making several requests to the same host, the underlying TCP + # connection will be reused, which can result in a significant + # performance increase (see HTTP persistent connection)." # - # NOTE: We use a separate requests.Session per scheme+hostname combination, - # in order to reuse connections to the same hostname to improve efficiency, - # but avoiding sharing state between different hosts-scheme combinations to - # minimize subtle security issues. Some cookies may not be HTTP-safe. + # NOTE: We use a separate requests.Session per scheme+hostname + # combination, in order to reuse connections to the same hostname to + # improve efficiency, but avoiding sharing state between different + # hosts-scheme combinations to minimize subtle security issues. + # Some cookies may not be HTTP-safe. self._sessions = {} def fetch(self, url, required_length): @@ -52,15 +53,17 @@ def fetch(self, url, required_length): Ensures the length of the downloaded data is up to 'required_length'. Arguments: - url: A URL string that represents a file location. - required_length: An integer value representing the file length in bytes. + url: A URL string that represents a file location. + required_length: An integer value representing the file length in + bytes. Raises: - tuf.exceptions.SlowRetrievalError: A timeout occurs while receiving data. - tuf.exceptions.FetcherHTTPError: An HTTP error code is received. + tuf.exceptions.SlowRetrievalError: A timeout occurs while receiving + data. + tuf.exceptions.FetcherHTTPError: An HTTP error code is received. Returns: - A bytes iterator + A bytes iterator """ # Get a customized session for each new schema+hostname combination. session = self._get_session(url) @@ -68,7 +71,8 @@ def fetch(self, url, required_length): # Get the requests.Response object for this URL. # # Defer downloading the response body with stream=True. - # Always set the timeout. This timeout value is interpreted by requests as: + # Always set the timeout. This timeout value is interpreted by + # requests as: # - connect timeout (max delay before first byte is received) # - read (gap) timeout (max delay between bytes received) response = session.get( @@ -82,18 +86,19 @@ def fetch(self, url, required_length): status = e.response.status_code raise tuf.exceptions.FetcherHTTPError(str(e), status) - # Define a generator function to be returned by fetch. This way the caller - # of fetch can differentiate between connection and actual data download - # and measure download times accordingly. + # Define a generator function to be returned by fetch. This way the + # caller of fetch can differentiate between connection and actual data + # download and measure download times accordingly. def chunks(): try: bytes_received = 0 while True: - # We download a fixed chunk of data in every round. This is so that we - # can defend against slow retrieval attacks. Furthermore, we do not - # wish to download an extremely large file in one shot. - # Before beginning the round, sleep (if set) for a short amount of - # time so that the CPU is not hogged in the while loop. + # We download a fixed chunk of data in every round. This is + # so that we can defend against slow retrieval attacks. + # Furthermore, we do not wish to download an extremely + # large file in one shot. Before beginning the round, sleep + # (if set) for a short amount of time so that the CPU is not + # hogged in the while loop. if tuf.settings.SLEEP_BEFORE_ROUND: time.sleep(tuf.settings.SLEEP_BEFORE_ROUND) @@ -102,13 +107,15 @@ def chunks(): required_length - bytes_received, ) - # NOTE: This may not handle some servers adding a Content-Encoding - # header, which may cause urllib3 to misbehave: + # NOTE: This may not handle some servers adding a + # Content-Encoding header, which may cause urllib3 to + # misbehave: # https://github.com/pypa/pip/blob/404838abcca467648180b358598c597b74d568c9/src/pip/_internal/download.py#L547-L582 data = response.raw.read(read_amount) bytes_received += len(data) - # We might have no more data to read. Check number of bytes downloaded. + # We might have no more data to read. Check number of bytes + # downloaded. if not data: logger.debug( "Downloaded " From 575c15b3396fdcad1bdc859716c159c783ce2910 Mon Sep 17 00:00:00 2001 From: Teodora Sechkova Date: Thu, 8 Apr 2021 13:02:24 +0300 Subject: [PATCH 05/11] Fix pylint W0212: protected-access Fix "Access to a protected member _mirror_target_download, _mirror_meta_download of a client class" Signed-off-by: Teodora Sechkova --- tuf/client_rework/mirrors.py | 4 ++-- tuf/client_rework/updater_rework.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tuf/client_rework/mirrors.py b/tuf/client_rework/mirrors.py index 0bdf07e2e6..91b9e87b4f 100644 --- a/tuf/client_rework/mirrors.py +++ b/tuf/client_rework/mirrors.py @@ -142,7 +142,7 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict): return list_of_mirrors -def _mirror_meta_download( +def mirror_meta_download( filename: str, upper_length: int, mirrors_config: Dict, @@ -171,7 +171,7 @@ def _mirror_meta_download( raise tuf.exceptions.NoWorkingMirrorError(file_mirror_errors) -def _mirror_target_download( +def mirror_target_download( fileinfo: str, mirrors_config: Dict, fetcher: "FetcherInterface" ) -> BinaryIO: """ diff --git a/tuf/client_rework/updater_rework.py b/tuf/client_rework/updater_rework.py index 8ddde1e4eb..ec783fa4b2 100644 --- a/tuf/client_rework/updater_rework.py +++ b/tuf/client_rework/updater_rework.py @@ -148,7 +148,7 @@ def download_target(self, target: Dict, destination_directory: str): The file is saved to the 'destination_directory' argument. """ try: - for temp_obj in mirrors._mirror_target_download( + for temp_obj in mirrors.mirror_target_download( target, self._mirrors, self._fetcher ): @@ -217,7 +217,7 @@ def _load_root(self) -> None: verified_root = None for next_version in range(lower_bound, upper_bound): try: - mirror_download = mirrors._mirror_meta_download( + mirror_download = mirrors.mirror_meta_download( self._get_relative_meta_name("root", version=next_version), settings.DEFAULT_ROOT_REQUIRED_LENGTH, self._mirrors, @@ -280,7 +280,7 @@ def _load_timestamp(self) -> None: TODO """ # TODO Check if timestamp exists locally - for temp_obj in mirrors._mirror_meta_download( + for temp_obj in mirrors.mirror_meta_download( "timestamp.json", settings.DEFAULT_TIMESTAMP_REQUIRED_LENGTH, self._mirrors, @@ -321,7 +321,7 @@ def _load_snapshot(self) -> None: # Check if exists locally # self.loadLocal('snapshot', snapshotVerifier) - for temp_obj in mirrors._mirror_meta_download( + for temp_obj in mirrors.mirror_meta_download( "snapshot.json", length, self._mirrors, self._fetcher ): @@ -360,7 +360,7 @@ def _load_targets(self, targets_role: str, parent_role: str) -> None: # Check if exists locally # self.loadLocal('snapshot', targetsVerifier) - for temp_obj in mirrors._mirror_meta_download( + for temp_obj in mirrors.mirror_meta_download( targets_role + ".json", length, self._mirrors, self._fetcher ): From 7a717f6179f6d24903c54ec94618e60790558190 Mon Sep 17 00:00:00 2001 From: Teodora Sechkova Date: Thu, 8 Apr 2021 14:02:25 +0300 Subject: [PATCH 06/11] Fix pylint C0103 and W0612 - fix C0103 invalid argument name STRICT_REQUIRED_LENGTH - use 'dummy' as an accepted by pylint unused variable name (W0612) Signed-off-by: Teodora Sechkova --- tuf/client_rework/download.py | 14 +++++++------- tuf/client_rework/mirrors.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tuf/client_rework/download.py b/tuf/client_rework/download.py index d04cc2baa3..0516d5e5ed 100644 --- a/tuf/client_rework/download.py +++ b/tuf/client_rework/download.py @@ -50,7 +50,7 @@ logger = logging.getLogger(__name__) -def download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True): +def download_file(url, required_length, fetcher, strict_required_length=True): """ Given the url and length of the desired file, this function opens a @@ -66,7 +66,7 @@ def download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True): required_length: An integer value representing the length of the file. - STRICT_REQUIRED_LENGTH: + strict_required_length: A Boolean indicator used to signal whether we should perform strict checking of required_length. True by default. We explicitly set this to False when we know that we want to turn this off for downloading the @@ -141,7 +141,7 @@ def download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True): _check_downloaded_length( number_of_bytes_received, required_length, - STRICT_REQUIRED_LENGTH=STRICT_REQUIRED_LENGTH, + strict_required_length=strict_required_length, average_download_speed=average_download_speed, ) @@ -158,7 +158,7 @@ def download_file(url, required_length, fetcher, STRICT_REQUIRED_LENGTH=True): def _check_downloaded_length( total_downloaded, required_length, - STRICT_REQUIRED_LENGTH=True, + strict_required_length=True, average_download_speed=None, ): """ @@ -178,7 +178,7 @@ def _check_downloaded_length( top-level roles. In both cases, 'required_length' is actually an upper limit on the length of the downloaded file. - STRICT_REQUIRED_LENGTH: + strict_required_length: A Boolean indicator used to signal whether we should perform strict checking of required_length. True by default. We explicitly set this to False when we know that we want to turn this off for downloading the @@ -192,7 +192,7 @@ def _check_downloaded_length( securesystemslib.exceptions.DownloadLengthMismatchError, if - STRICT_REQUIRED_LENGTH is True and total_downloaded is not equal + strict_required_length is True and total_downloaded is not equal required_length. tuf.exceptions.SlowRetrievalError, if the total downloaded was @@ -214,7 +214,7 @@ def _check_downloaded_length( # What we downloaded is not equal to the required length, but did we ask # for strict checking of required length? - if STRICT_REQUIRED_LENGTH: + if strict_required_length: logger.info( "Downloaded " + str(total_downloaded) + " bytes, but" " expected " diff --git a/tuf/client_rework/mirrors.py b/tuf/client_rework/mirrors.py index 91b9e87b4f..debf772f34 100644 --- a/tuf/client_rework/mirrors.py +++ b/tuf/client_rework/mirrors.py @@ -110,7 +110,7 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict): in_confined_directory = securesystemslib.util.file_in_confined_directories list_of_mirrors = [] - for junk, mirror_info in six.iteritems(mirrors_dict): + for dummy, mirror_info in six.iteritems(mirrors_dict): # Does mirror serve this file type at all? path = mirror_info.get(path_key) if path is None: @@ -157,7 +157,7 @@ def mirror_meta_download( for file_mirror in file_mirrors: try: temp_obj = download.download_file( - file_mirror, upper_length, fetcher, STRICT_REQUIRED_LENGTH=False + file_mirror, upper_length, fetcher, strict_required_length=False ) temp_obj.seek(0) From 1d2721df80aeee860f98b3722e633665a4902ab4 Mon Sep 17 00:00:00 2001 From: Teodora Sechkova Date: Thu, 8 Apr 2021 14:08:20 +0300 Subject: [PATCH 07/11] Fix pylint R1720: Unnecessary "else" after "raise" Signed-off-by: Teodora Sechkova --- tuf/client_rework/download.py | 56 ++++++++++++++++------------------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/tuf/client_rework/download.py b/tuf/client_rework/download.py index 0516d5e5ed..ec574c3f7d 100644 --- a/tuf/client_rework/download.py +++ b/tuf/client_rework/download.py @@ -131,11 +131,10 @@ def download_file(url, required_length, fetcher, strict_required_length=True): ) break - else: - logger.debug( - "The average download speed has not dipped below the" - " minimum average download speed set in tuf.settings.py." - ) + logger.debug( + "The average download speed has not dipped below the" + " minimum average download speed set in tuf.settings.py." + ) # Does the total number of downloaded bytes match the required length? _check_downloaded_length( @@ -236,34 +235,31 @@ def _check_downloaded_length( if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: raise tuf.exceptions.SlowRetrievalError(average_download_speed) - else: - logger.debug( - "Good average download speed: " - + repr(average_download_speed) - + " bytes per second" - ) + logger.debug( + "Good average download speed: " + + repr(average_download_speed) + + " bytes per second" + ) raise tuf.exceptions.DownloadLengthMismatchError( required_length, total_downloaded ) - else: - # We specifically disabled strict checking of required length, but - # we will log a warning anyway. This is useful when we wish to - # download the Timestamp or Root metadata, for which we have no - # signed metadata; so, we must guess a reasonable required_length - # for it. - if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: - raise tuf.exceptions.SlowRetrievalError(average_download_speed) - - else: - logger.debug( - "Good average download speed: " - + repr(average_download_speed) - + " bytes per second" - ) + # We specifically disabled strict checking of required length, but + # we will log a warning anyway. This is useful when we wish to + # download the Timestamp or Root metadata, for which we have no + # signed metadata; so, we must guess a reasonable required_length + # for it. + if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: + raise tuf.exceptions.SlowRetrievalError(average_download_speed) + + logger.debug( + "Good average download speed: " + + repr(average_download_speed) + + " bytes per second" + ) - logger.info( - "Downloaded " + str(total_downloaded) + " bytes out of an" - " upper limit of " + str(required_length) + " bytes." - ) + logger.info( + "Downloaded " + str(total_downloaded) + " bytes out of an" + " upper limit of " + str(required_length) + " bytes." + ) From aaedcfe56261a607a828489f492c09ede6e171af Mon Sep 17 00:00:00 2001 From: Teodora Sechkova Date: Thu, 8 Apr 2021 14:11:18 +0300 Subject: [PATCH 08/11] Fix pylint W0703: broad-except pylint cannot figure out that we store the exceptions in a dictionary to raise them later so we disable the warning. This should be reviewed in the future still. Signed-off-by: Teodora Sechkova --- tuf/client_rework/mirrors.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tuf/client_rework/mirrors.py b/tuf/client_rework/mirrors.py index debf772f34..7177809d38 100644 --- a/tuf/client_rework/mirrors.py +++ b/tuf/client_rework/mirrors.py @@ -163,7 +163,10 @@ def mirror_meta_download( temp_obj.seek(0) yield temp_obj - except Exception as exception: + # pylint cannot figure out that we store the exceptions + # in a dictionary to raise them later so we disable + # the warning. This should be reviewed in the future still. + except Exception as exception: # pylint: disable=broad-except file_mirror_errors[file_mirror] = exception finally: @@ -192,7 +195,10 @@ def mirror_target_download( temp_obj.seek(0) yield temp_obj - except Exception as exception: + # pylint cannot figure out that we store the exceptions + # in a dictionary to raise them later so we disable + # the warning. This should be reviewed in the future still. + except Exception as exception: # pylint: disable=broad-except file_mirror_errors[file_mirror] = exception finally: From 2f0bbd04f636b77f43b07782881c638058d71e86 Mon Sep 17 00:00:00 2001 From: Teodora Sechkova Date: Thu, 8 Apr 2021 14:23:19 +0300 Subject: [PATCH 09/11] Remove use of future and six six and future are Python 2 compatibility modules which are no longer needed after the end of Python 2 support. Signed-off-by: Teodora Sechkova --- tuf/client_rework/download.py | 15 ++------------- tuf/client_rework/mirrors.py | 17 +++-------------- tuf/client_rework/requests_fetcher.py | 4 ++-- 3 files changed, 7 insertions(+), 29 deletions(-) diff --git a/tuf/client_rework/download.py b/tuf/client_rework/download.py index ec574c3f7d..ac097edcf3 100644 --- a/tuf/client_rework/download.py +++ b/tuf/client_rework/download.py @@ -23,24 +23,13 @@ metadata of that file. """ - -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import ( - absolute_import, - division, - print_function, - unicode_literals, -) - import logging import tempfile import timeit +from urllib import parse import securesystemslib import securesystemslib.util -import six import tuf import tuf.exceptions @@ -98,7 +87,7 @@ def download_file(url, required_length, fetcher, strict_required_length=True): # a url with their single-character equivalent. A back-slash may be # encoded as %5c in the url, which should also be replaced with a forward # slash. - url = six.moves.urllib.parse.unquote(url).replace("\\", "/") + url = parse.unquote(url).replace("\\", "/") logger.info("Downloading: " + repr(url)) # This is the temporary file that we will return to contain the contents of diff --git a/tuf/client_rework/mirrors.py b/tuf/client_rework/mirrors.py index 7177809d38..311416dc75 100644 --- a/tuf/client_rework/mirrors.py +++ b/tuf/client_rework/mirrors.py @@ -22,22 +22,11 @@ of the file with respect to the base url. """ - -# Help with Python 3 compatibility, where the print statement is a function, an -# implicit relative import is invalid, and the '/' operator performs true -# division. Example: print 'hello world' raises a 'SyntaxError' exception. -from __future__ import ( - absolute_import, - division, - print_function, - unicode_literals, -) - import os from typing import BinaryIO, Dict, TextIO +from urllib import parse import securesystemslib -import six import tuf import tuf.client_rework.download as download @@ -110,7 +99,7 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict): in_confined_directory = securesystemslib.util.file_in_confined_directories list_of_mirrors = [] - for dummy, mirror_info in six.iteritems(mirrors_dict): + for mirror_info in mirrors_dict.values(): # Does mirror serve this file type at all? path = mirror_info.get(path_key) if path is None: @@ -131,7 +120,7 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict): # on the server side. Do *NOT* pass URLs with Unicode characters without # first encoding the URL as UTF-8. Needed a long-term solution with #61. # http://bugs.python.org/issue1712522 - file_path = six.moves.urllib.parse.quote(file_path) + file_path = parse.quote(file_path) url = os.path.join(mirror_info["url_prefix"], path, file_path) # The above os.path.join() result as well as input file_path may be diff --git a/tuf/client_rework/requests_fetcher.py b/tuf/client_rework/requests_fetcher.py index 545a23feff..fadd022743 100644 --- a/tuf/client_rework/requests_fetcher.py +++ b/tuf/client_rework/requests_fetcher.py @@ -7,10 +7,10 @@ import logging import time +from urllib import parse # Imports import requests -import six import urllib3.exceptions import tuf.exceptions @@ -147,7 +147,7 @@ def _get_session(self, url): """ # Use a different requests.Session per schema+hostname combination, to # reuse connections while minimizing subtle security issues. - parsed_url = six.moves.urllib.parse.urlparse(url) + parsed_url = parse.urlparse(url) if not parsed_url.scheme or not parsed_url.hostname: raise tuf.exceptions.URLParsingError( From 546bb785f9461e10400a7f4c7e36368ebd3b5cc9 Mon Sep 17 00:00:00 2001 From: Teodora Sechkova Date: Thu, 8 Apr 2021 14:45:24 +0300 Subject: [PATCH 10/11] Fix client imports Fix imports to be vendoring compatible. Signed-off-by: Teodora Sechkova --- tuf/client_rework/download.py | 20 +++++++++----------- tuf/client_rework/metadata_wrapper.py | 8 ++++---- tuf/client_rework/mirrors.py | 23 ++++++++++++----------- tuf/client_rework/requests_fetcher.py | 22 +++++++++++----------- tuf/client_rework/updater_rework.py | 2 +- 5 files changed, 37 insertions(+), 38 deletions(-) diff --git a/tuf/client_rework/download.py b/tuf/client_rework/download.py index ac097edcf3..858355523f 100644 --- a/tuf/client_rework/download.py +++ b/tuf/client_rework/download.py @@ -28,12 +28,10 @@ import timeit from urllib import parse -import securesystemslib -import securesystemslib.util +from securesystemslib import formats as sslib_formats import tuf -import tuf.exceptions -import tuf.formats +from tuf import exceptions, formats # See 'log.py' to learn how logging is handled in TUF. logger = logging.getLogger(__name__) @@ -65,7 +63,7 @@ def download_file(url, required_length, fetcher, strict_required_length=True): A file object is created on disk to store the contents of 'url'. - tuf.exceptions.DownloadLengthMismatchError, if there was a + exceptions.DownloadLengthMismatchError, if there was a mismatch of observed vs expected lengths while downloading the file. securesystemslib.exceptions.FormatError, if any of the arguments are @@ -78,8 +76,8 @@ def download_file(url, required_length, fetcher, strict_required_length=True): """ # Do all of the arguments have the appropriate format? # Raise 'securesystemslib.exceptions.FormatError' if there is a mismatch. - securesystemslib.formats.URL_SCHEMA.check_match(url) - tuf.formats.LENGTH_SCHEMA.check_match(required_length) + sslib_formats.URL_SCHEMA.check_match(url) + formats.LENGTH_SCHEMA.check_match(required_length) # 'url.replace('\\', '/')' is needed for compatibility with Windows-based # systems, because they might use back-slashes in place of forward-slashes. @@ -183,7 +181,7 @@ def _check_downloaded_length( strict_required_length is True and total_downloaded is not equal required_length. - tuf.exceptions.SlowRetrievalError, if the total downloaded was + exceptions.SlowRetrievalError, if the total downloaded was done in less than the acceptable download speed (as set in tuf.settings.py). @@ -222,7 +220,7 @@ def _check_downloaded_length( ) if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: - raise tuf.exceptions.SlowRetrievalError(average_download_speed) + raise exceptions.SlowRetrievalError(average_download_speed) logger.debug( "Good average download speed: " @@ -230,7 +228,7 @@ def _check_downloaded_length( + " bytes per second" ) - raise tuf.exceptions.DownloadLengthMismatchError( + raise exceptions.DownloadLengthMismatchError( required_length, total_downloaded ) @@ -240,7 +238,7 @@ def _check_downloaded_length( # signed metadata; so, we must guess a reasonable required_length # for it. if average_download_speed < tuf.settings.MIN_AVERAGE_DOWNLOAD_SPEED: - raise tuf.exceptions.SlowRetrievalError(average_download_speed) + raise exceptions.SlowRetrievalError(average_download_speed) logger.debug( "Good average download speed: " diff --git a/tuf/client_rework/metadata_wrapper.py b/tuf/client_rework/metadata_wrapper.py index 6f182dc336..18f0d6d9aa 100644 --- a/tuf/client_rework/metadata_wrapper.py +++ b/tuf/client_rework/metadata_wrapper.py @@ -9,7 +9,7 @@ from securesystemslib.keys import format_metadata_to_key -import tuf.exceptions +from tuf import exceptions, formats from tuf.api import metadata @@ -64,7 +64,7 @@ def verify(self, keys, threshold): verified += 1 if verified < threshold: - raise tuf.exceptions.InsufficientKeysError + raise exceptions.InsufficientKeysError def persist(self, filename): """ @@ -77,13 +77,13 @@ def expires(self, reference_time=None): TODO """ if reference_time is None: - expires_timestamp = tuf.formats.datetime_to_unix_timestamp( + expires_timestamp = formats.datetime_to_unix_timestamp( self._meta.signed.expires ) reference_time = int(time.time()) if expires_timestamp < reference_time: - raise tuf.exceptions.ExpiredMetadataError + raise exceptions.ExpiredMetadataError class RootWrapper(MetadataWrapper): diff --git a/tuf/client_rework/mirrors.py b/tuf/client_rework/mirrors.py index 311416dc75..97962e9eb7 100644 --- a/tuf/client_rework/mirrors.py +++ b/tuf/client_rework/mirrors.py @@ -26,11 +26,12 @@ from typing import BinaryIO, Dict, TextIO from urllib import parse -import securesystemslib +from securesystemslib import exceptions as sslib_exceptions +from securesystemslib import formats as sslib_formats +from securesystemslib import util as sslib_util -import tuf -import tuf.client_rework.download as download -import tuf.formats +from tuf import exceptions, formats +from tuf.client_rework import download # The type of file to be downloaded from a repository. The # 'get_list_of_mirrors' function supports these file types. @@ -78,13 +79,13 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict): """ # Checking if all the arguments have appropriate format. - tuf.formats.RELPATH_SCHEMA.check_match(file_path) - tuf.formats.MIRRORDICT_SCHEMA.check_match(mirrors_dict) - securesystemslib.formats.NAME_SCHEMA.check_match(file_type) + formats.RELPATH_SCHEMA.check_match(file_path) + formats.MIRRORDICT_SCHEMA.check_match(mirrors_dict) + sslib_formats.NAME_SCHEMA.check_match(file_type) # Verify 'file_type' is supported. if file_type not in _SUPPORTED_FILE_TYPES: - raise securesystemslib.exceptions.Error( + raise sslib_exceptions.Error( "Invalid file_type argument." " Supported file types: " + repr(_SUPPORTED_FILE_TYPES) ) @@ -96,7 +97,7 @@ def get_list_of_mirrors(file_type, file_path, mirrors_dict): # on a repository mirror when fetching target files. This field may be set # by the client when the repository mirror is added to the # 'tuf.client.updater.Updater' object. - in_confined_directory = securesystemslib.util.file_in_confined_directories + in_confined_directory = sslib_util.file_in_confined_directories list_of_mirrors = [] for mirror_info in mirrors_dict.values(): @@ -160,7 +161,7 @@ def mirror_meta_download( finally: if file_mirror_errors: - raise tuf.exceptions.NoWorkingMirrorError(file_mirror_errors) + raise exceptions.NoWorkingMirrorError(file_mirror_errors) def mirror_target_download( @@ -192,4 +193,4 @@ def mirror_target_download( finally: if file_mirror_errors: - raise tuf.exceptions.NoWorkingMirrorError(file_mirror_errors) + raise exceptions.NoWorkingMirrorError(file_mirror_errors) diff --git a/tuf/client_rework/requests_fetcher.py b/tuf/client_rework/requests_fetcher.py index fadd022743..ef18233024 100644 --- a/tuf/client_rework/requests_fetcher.py +++ b/tuf/client_rework/requests_fetcher.py @@ -13,8 +13,8 @@ import requests import urllib3.exceptions -import tuf.exceptions -import tuf.settings +import tuf +from tuf import exceptions, settings from tuf.client_rework.fetcher import FetcherInterface # Globals @@ -58,9 +58,9 @@ def fetch(self, url, required_length): bytes. Raises: - tuf.exceptions.SlowRetrievalError: A timeout occurs while receiving + exceptions.SlowRetrievalError: A timeout occurs while receiving data. - tuf.exceptions.FetcherHTTPError: An HTTP error code is received. + exceptions.FetcherHTTPError: An HTTP error code is received. Returns: A bytes iterator @@ -76,7 +76,7 @@ def fetch(self, url, required_length): # - connect timeout (max delay before first byte is received) # - read (gap) timeout (max delay between bytes received) response = session.get( - url, stream=True, timeout=tuf.settings.SOCKET_TIMEOUT + url, stream=True, timeout=settings.SOCKET_TIMEOUT ) # Check response status. try: @@ -84,7 +84,7 @@ def fetch(self, url, required_length): except requests.HTTPError as e: response.close() status = e.response.status_code - raise tuf.exceptions.FetcherHTTPError(str(e), status) + raise exceptions.FetcherHTTPError(str(e), status) # Define a generator function to be returned by fetch. This way the # caller of fetch can differentiate between connection and actual data @@ -99,11 +99,11 @@ def chunks(): # large file in one shot. Before beginning the round, sleep # (if set) for a short amount of time so that the CPU is not # hogged in the while loop. - if tuf.settings.SLEEP_BEFORE_ROUND: - time.sleep(tuf.settings.SLEEP_BEFORE_ROUND) + if settings.SLEEP_BEFORE_ROUND: + time.sleep(settings.SLEEP_BEFORE_ROUND) read_amount = min( - tuf.settings.CHUNK_SIZE, + settings.CHUNK_SIZE, required_length - bytes_received, ) @@ -134,7 +134,7 @@ def chunks(): break except urllib3.exceptions.ReadTimeoutError as e: - raise tuf.exceptions.SlowRetrievalError(str(e)) + raise exceptions.SlowRetrievalError(str(e)) finally: response.close() @@ -150,7 +150,7 @@ def _get_session(self, url): parsed_url = parse.urlparse(url) if not parsed_url.scheme or not parsed_url.hostname: - raise tuf.exceptions.URLParsingError( + raise exceptions.URLParsingError( "Could not get scheme and hostname from URL: " + url ) diff --git a/tuf/client_rework/updater_rework.py b/tuf/client_rework/updater_rework.py index ec783fa4b2..078f17304e 100644 --- a/tuf/client_rework/updater_rework.py +++ b/tuf/client_rework/updater_rework.py @@ -18,7 +18,7 @@ from tuf import exceptions, settings from tuf.client.fetcher import FetcherInterface -from tuf.client_rework import download, mirrors, requests_fetcher +from tuf.client_rework import mirrors, requests_fetcher from .metadata_wrapper import ( RootWrapper, From 101ab3d62b80c4f95753d2faf0fbe650de6b0012 Mon Sep 17 00:00:00 2001 From: Teodora Sechkova Date: Wed, 14 Apr 2021 13:32:03 +0300 Subject: [PATCH 11/11] Disable pylint W1201: logging-not-lazy Disable pylint's "Use lazy % formatting in logging functions" warning until a common logging approach is decided. See #1334. Signed-off-by: Teodora Sechkova --- tuf/api/pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/tuf/api/pylintrc b/tuf/api/pylintrc index 409a96149f..23cfce8aea 100644 --- a/tuf/api/pylintrc +++ b/tuf/api/pylintrc @@ -14,6 +14,7 @@ disable=fixme, too-few-public-methods, too-many-arguments, + logging-not-lazy, [BASIC] good-names=i,j,k,v,e,f,fn,fp,_type