From 5eda7585514a036c994ef9580cdb1d879f7ae74d Mon Sep 17 00:00:00 2001 From: Ina Panova Date: Wed, 11 Mar 2020 20:44:20 +0100 Subject: [PATCH] Add secret pull option. closes #6311 https://pulp.plan.io/issues/6311 --- common/pulp_docker/common/constants.py | 1 + docs/tech-reference/importer.rst | 3 ++ .../pulp_docker/extensions/admin/cudl.py | 13 ++++++ plugins/pulp_docker/plugins/importers/sync.py | 3 +- plugins/pulp_docker/plugins/registry.py | 23 ++++++++-- plugins/test/unit/plugins/test_registry.py | 46 +++++++++++++------ 6 files changed, 70 insertions(+), 19 deletions(-) diff --git a/common/pulp_docker/common/constants.py b/common/pulp_docker/common/constants.py index c6156d9e..b51e4138 100644 --- a/common/pulp_docker/common/constants.py +++ b/common/pulp_docker/common/constants.py @@ -19,6 +19,7 @@ # Config keys for the importer CONFIG_KEY_UPSTREAM_NAME = 'upstream_name' +CONFIG_KEY_PULL_SECRET = 'pull_secret' # Config keys for the distributor plugin conf CONFIG_KEY_DOCKER_PUBLISH_DIRECTORY = 'docker_publish_directory' diff --git a/docs/tech-reference/importer.rst b/docs/tech-reference/importer.rst index e9ddcb7a..c76549f6 100644 --- a/docs/tech-reference/importer.rst +++ b/docs/tech-reference/importer.rst @@ -36,3 +36,6 @@ The following options are available to the docker importer configuration. ``upstream_name`` The name of the repository to import from the upstream repository. + +``pull_secret`` +Full path to the json file that contains pull secret. diff --git a/extensions_admin/pulp_docker/extensions/admin/cudl.py b/extensions_admin/pulp_docker/extensions/admin/cudl.py index 581bec89..1252a82e 100644 --- a/extensions_admin/pulp_docker/extensions/admin/cudl.py +++ b/extensions_admin/pulp_docker/extensions/admin/cudl.py @@ -45,6 +45,9 @@ d = _('name of the upstream repository') OPT_UPSTREAM_NAME = PulpCliOption('--upstream-name', d, required=False) +d = _('full path to the json file that contains pull secret') +OPT_PULL_SECRET = PulpCliOption('--pull-secret', d, required=False) + d = _('Enable sync of v1 API. defaults to "false." DEPRECATED.') OPT_ENABLE_V1 = PulpCliOption('--enable-v1', d, required=False, parse_func=okaara_parsers.parse_boolean) @@ -93,6 +96,10 @@ def _parse_importer_config(self, user_input): if whitelist_tags is not None: config[constants.CONFIG_KEY_WHITELIST_TAGS] = whitelist_tags + pull_secret = user_input.pop(OPT_PULL_SECRET.keyword, None) + if pull_secret is not None: + config[constants.CONFIG_KEY_PULL_SECRET] = pull_secret + return config @@ -112,6 +119,7 @@ def __init__(self, context): self.add_option(OPT_ENABLE_V1) self.add_option(OPT_ENABLE_V2) self.add_option(OPTION_WHITELIST_TAGS) + self.add_option(OPT_PULL_SECRET) self.sync_group.add_option(OPT_UPSTREAM_NAME) self.options_bundle.opt_feed.description = DESC_FEED @@ -171,6 +179,7 @@ def __init__(self, context): self.add_option(OPT_ENABLE_V1) self.add_option(OPT_ENABLE_V2) self.add_option(OPTION_WHITELIST_TAGS) + self.add_option(OPT_PULL_SECRET) self.sync_group.add_option(OPT_UPSTREAM_NAME) self.options_bundle.opt_feed.description = DESC_FEED @@ -183,6 +192,10 @@ def run(self, **kwargs): if name is not None: importer_config[constants.CONFIG_KEY_UPSTREAM_NAME] = name + pull_secret = kwargs.pop(OPT_PULL_SECRET.keyword, None) + if pull_secret is not None: + importer_config[constants.CONFIG_KEY_PULL_SECRET] = pull_secret + if importer_config: whitelist_tags = importer_config.get(OPTION_WHITELIST_TAGS.keyword, None) diff --git a/plugins/pulp_docker/plugins/importers/sync.py b/plugins/pulp_docker/plugins/importers/sync.py index 93fb9baa..a236f7d9 100644 --- a/plugins/pulp_docker/plugins/importers/sync.py +++ b/plugins/pulp_docker/plugins/importers/sync.py @@ -65,7 +65,7 @@ def __init__(self, repo=None, conduit=None, config=None): # Create a Repository object to interact with. self.index_repository = registry.V2Repository( - upstream_name, download_config, url, self.get_working_dir()) + upstream_name, download_config, url, self.get_working_dir(), config) self.v1_index_repository = registry.V1Repository(upstream_name, download_config, url, self.get_working_dir()) @@ -420,6 +420,7 @@ def __init__(self, step_type, downloads=None, repo=None, conduit=None, config=No importer_constants.KEY_BASIC_AUTH_USER, None) self.basic_auth_password = config.repo_plugin_config.pop( importer_constants.KEY_BASIC_AUTH_PASS, None) + super(AuthDownloadStep, self).__init__( step_type, downloads=downloads, repo=repo, conduit=conduit, config=config, working_dir=working_dir, plugin_type=plugin_type) diff --git a/plugins/pulp_docker/plugins/registry.py b/plugins/pulp_docker/plugins/registry.py index 036ea3a0..5035641c 100644 --- a/plugins/pulp_docker/plugins/registry.py +++ b/plugins/pulp_docker/plugins/registry.py @@ -9,6 +9,7 @@ import re import traceback import urlparse +import base64 from nectar.downloaders.threaded import HTTPThreadedDownloader from nectar.listener import AggregatingEventListener @@ -279,7 +280,7 @@ class V2Repository(object): MANIFEST_PATH = '/v2/{name}/manifests/{reference}' TAGS_PATH = '/v2/{name}/tags/list' - def __init__(self, name, download_config, registry_url, working_dir): + def __init__(self, name, download_config, registry_url, working_dir, config): """ Initialize the V2Repository. @@ -305,11 +306,27 @@ def __init__(self, name, download_config, registry_url, working_dir): self.download_config = download_config self.registry_url = registry_url + self.repo_config = config + + user = None + passw = None + pull_secret_file = self.repo_config.get(constants.CONFIG_KEY_PULL_SECRET, None) + if pull_secret_file: + with open(pull_secret_file, 'r') as pull_secret: + pull_secret = json.load(pull_secret) + parse = urlparse.urlsplit(self.registry_url) + registries = pull_secret['auths'] + if parse.netloc in registries: + creds = base64.b64decode(registries[parse.netloc]['auth']) + user, passw = creds.split(':') # Use basic auth information for retrieving tokens from auth server and for downloading # with basic auth - self.auth_downloader = HTTPThreadedDownloader(copy.deepcopy(self.download_config), - AggregatingEventListener()) + dc = copy.deepcopy(self.download_config) + if user and passw: + dc.basic_auth_username = user + dc.basic_auth_password = passw + self.auth_downloader = HTTPThreadedDownloader(dc, AggregatingEventListener()) self.download_config.basic_auth_username = None self.download_config.basic_auth_password = None self.downloader = HTTPThreadedDownloader(self.download_config, AggregatingEventListener()) diff --git a/plugins/test/unit/plugins/test_registry.py b/plugins/test/unit/plugins/test_registry.py index 9aa9a1f9..0796ce35 100644 --- a/plugins/test/unit/plugins/test_registry.py +++ b/plugins/test/unit/plugins/test_registry.py @@ -373,7 +373,8 @@ def test___init__(self): download_config = DownloaderConfig(max_concurrent=25) registry_url = 'https://registry.example.com' working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) self.assertEqual(r.name, name) self.assertEqual(r.download_config, download_config) @@ -381,6 +382,7 @@ def test___init__(self): self.assertEqual(type(r.downloader), HTTPThreadedDownloader) self.assertEqual(r.downloader.config, download_config) self.assertEqual(r.working_dir, working_dir) + self.assertEqual(r.repo_config, config) def test_api_version_check_incorrect_header(self): """ @@ -403,7 +405,8 @@ def download_one(request): download_config = DownloaderConfig(max_concurrent=25) registry_url = 'https://registry.example.com' working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) r.downloader.download_one = mock.MagicMock(side_effect=download_one) self.assertFalse(r.api_version_check()) @@ -417,7 +420,8 @@ def test_api_version_check_ioerror(self, mock_get_path): download_config = DownloaderConfig(max_concurrent=25) registry_url = 'https://registry.example.com' working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) self.assertFalse(r.api_version_check()) @@ -444,7 +448,8 @@ def download_one(request): download_config = DownloaderConfig(max_concurrent=25) registry_url = 'https://registry.example.com' working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) r.downloader.download_one = mock.MagicMock(side_effect=download_one) # This should not raise an Exception @@ -470,7 +475,8 @@ def download_one(request): download_config = DownloaderConfig(max_concurrent=25) registry_url = 'https://registry.example.com' working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) r.downloader.download_one = mock.MagicMock(side_effect=download_one) # This should not raise an Exception @@ -493,7 +499,8 @@ def test_create_blob_download_request(self): download_config = DownloaderConfig(max_concurrent=25) registry_url = 'https://registry.example.com' working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) digest = 'sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef' request = r.create_blob_download_request(digest) @@ -526,7 +533,8 @@ def download_one(request): download_config = DownloaderConfig(max_concurrent=25) registry_url = 'https://registry.example.com' working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) r.downloader.download_one = mock.MagicMock(side_effect=download_one) digest = 'sha256:46356a7d9575b4cee21e7867b1b83a51788610b7719a616096d943b44737ad9a' with open(os.path.join(TEST_DATA_PATH, 'manifest_repeated_layers.json')) as manifest_file: @@ -557,7 +565,8 @@ def download_one(request): download_config = DownloaderConfig(max_concurrent=25) registry_url = 'https://registry.example.com' working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) r.downloader.download_one = mock.MagicMock(side_effect=download_one) tags = r.get_tags() @@ -573,7 +582,8 @@ def test_get_tags_failed(self, mock_download_one): download_config = DownloaderConfig() registry_url = 'https://registry.example.com' working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) with self.assertRaises(PulpCodedException) as assertion: r.get_tags() @@ -590,7 +600,8 @@ def test__get_path_failed(self, mock_download_one, mock_request_token): download_config = DownloaderConfig(max_concurrent=25) registry_url = 'https://registry.example.com' working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) report = DownloadReport(registry_url + '/some/path', StringIO()) report.error_report['response_code'] = httplib.UNAUTHORIZED @@ -621,7 +632,8 @@ def download_one(request): download_config = DownloaderConfig(max_concurrent=25) registry_url = 'https://registry.example.com' working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) r.downloader.download_one = mock.MagicMock(side_effect=download_one) headers, body = r._get_path('/some/path') @@ -666,7 +678,8 @@ def test_dockerhub_v2_registry_without_namespace(self, http_threaded_downloader) registry_url = "https://registry-1.docker.io" download_config = mock.MagicMock() working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) self.assertEqual('library/test_image', r.name, "Non-name-spaced image not prepended") @mock.patch('pulp_docker.plugins.registry.HTTPThreadedDownloader') @@ -675,7 +688,8 @@ def test_dockerhub_v2_registry_with_namespace(self, http_threaded_downloader): registry_url = "https://registry-1.docker.io" download_config = mock.MagicMock() working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) self.assertNotEqual('library/library/test_image', r.name, "Name-spaced image prepended with library") @@ -685,7 +699,8 @@ def test_non_dockerhub_v2_registry_with_namespace(self, http_threaded_downloader registry_url = "https://somewhere.not-docker.io" download_config = mock.MagicMock() working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) self.assertNotEqual('library/library/test_image', r.name, "Name-spaced Non-docker hub image prepended with library") @@ -695,6 +710,7 @@ def test_non_dockerhub_v2_registry_without_namespace(self, http_threaded_downloa registry_url = "https://somewhere.not-docker.io" download_config = mock.MagicMock() working_dir = '/a/working/dir' - r = registry.V2Repository(name, download_config, registry_url, working_dir) + config = {} + r = registry.V2Repository(name, download_config, registry_url, working_dir, config) self.assertNotEqual('library/test_image', r.name, "Non-docker hub image prepended with library")