Add pull secret option.

common/pulp_docker/common/constants.py

@@ -19,6 +19,7 @@
 
 # Config keys for the importer
 CONFIG_KEY_UPSTREAM_NAME = 'upstream_name'
+CONFIG_KEY_PULL_SECRET = 'pull_secret'
 
 # Config keys for the distributor plugin conf
 CONFIG_KEY_DOCKER_PUBLISH_DIRECTORY = 'docker_publish_directory'

docs/tech-reference/importer.rst

@@ -36,3 +36,6 @@ The following options are available to the docker importer configuration.
 
 ``upstream_name``
  The name of the repository to import from the upstream repository.
+
+``pull_secret``
+Full path to the json file that contains pull secret.

extensions_admin/pulp_docker/extensions/admin/cudl.py

@@ -45,6 +45,9 @@
 d = _('name of the upstream repository')
 OPT_UPSTREAM_NAME = PulpCliOption('--upstream-name', d, required=False)
 
+d = _('full path to the json file that contains pull secret')
+OPT_PULL_SECRET = PulpCliOption('--pull-secret', d, required=False)
+
 d = _('Enable sync of v1 API. defaults to "false." DEPRECATED.')
 OPT_ENABLE_V1 = PulpCliOption('--enable-v1', d, required=False,
                               parse_func=okaara_parsers.parse_boolean)
@@ -93,6 +96,10 @@ def _parse_importer_config(self, user_input):
         if whitelist_tags is not None:
             config[constants.CONFIG_KEY_WHITELIST_TAGS] = whitelist_tags
 
+        pull_secret = user_input.pop(OPT_PULL_SECRET.keyword, None)
+        if pull_secret is not None:
+            config[constants.CONFIG_KEY_PULL_SECRET] = pull_secret
+
         return config
 
 
@@ -112,6 +119,7 @@ def __init__(self, context):
         self.add_option(OPT_ENABLE_V1)
         self.add_option(OPT_ENABLE_V2)
         self.add_option(OPTION_WHITELIST_TAGS)
+        self.add_option(OPT_PULL_SECRET)
         self.sync_group.add_option(OPT_UPSTREAM_NAME)
         self.options_bundle.opt_feed.description = DESC_FEED
 
@@ -171,6 +179,7 @@ def __init__(self, context):
         self.add_option(OPT_ENABLE_V1)
         self.add_option(OPT_ENABLE_V2)
         self.add_option(OPTION_WHITELIST_TAGS)
+        self.add_option(OPT_PULL_SECRET)
         self.sync_group.add_option(OPT_UPSTREAM_NAME)
         self.options_bundle.opt_feed.description = DESC_FEED
 
@@ -183,6 +192,10 @@ def run(self, **kwargs):
         if name is not None:
             importer_config[constants.CONFIG_KEY_UPSTREAM_NAME] = name
 
+        pull_secret = kwargs.pop(OPT_PULL_SECRET.keyword, None)
+        if pull_secret is not None:
+            importer_config[constants.CONFIG_KEY_PULL_SECRET] = pull_secret
+
         if importer_config:
             whitelist_tags = importer_config.get(OPTION_WHITELIST_TAGS.keyword, None)
 

plugins/pulp_docker/plugins/importers/sync.py

@@ -65,7 +65,7 @@ def __init__(self, repo=None, conduit=None, config=None):
 
         # Create a Repository object to interact with.
         self.index_repository = registry.V2Repository(
-            upstream_name, download_config, url, self.get_working_dir())
+            upstream_name, download_config, url, self.get_working_dir(), config)
         self.v1_index_repository = registry.V1Repository(upstream_name, download_config, url,
                                                          self.get_working_dir())
 
@@ -420,6 +420,7 @@ def __init__(self, step_type, downloads=None, repo=None, conduit=None, config=No
             importer_constants.KEY_BASIC_AUTH_USER, None)
         self.basic_auth_password = config.repo_plugin_config.pop(
             importer_constants.KEY_BASIC_AUTH_PASS, None)
+
         super(AuthDownloadStep, self).__init__(
             step_type, downloads=downloads, repo=repo, conduit=conduit, config=config,
             working_dir=working_dir, plugin_type=plugin_type)

plugins/pulp_docker/plugins/registry.py

@@ -9,6 +9,7 @@
 import re
 import traceback
 import urlparse
+import base64
 
 from nectar.downloaders.threaded import HTTPThreadedDownloader
 from nectar.listener import AggregatingEventListener
@@ -279,7 +280,7 @@ class V2Repository(object):
     MANIFEST_PATH = '/v2/{name}/manifests/{reference}'
     TAGS_PATH = '/v2/{name}/tags/list'
 
-    def __init__(self, name, download_config, registry_url, working_dir):
+    def __init__(self, name, download_config, registry_url, working_dir, config):
         """
         Initialize the V2Repository.
 
@@ -305,11 +306,27 @@ def __init__(self, name, download_config, registry_url, working_dir):
 
         self.download_config = download_config
         self.registry_url = registry_url
+        self.repo_config = config
+
+        user = None
+        passw = None
+        pull_secret_file = self.repo_config.get(constants.CONFIG_KEY_PULL_SECRET, None)
+        if pull_secret_file:
+            with open(pull_secret_file, 'r') as pull_secret:
+                pull_secret = json.load(pull_secret)
+            parse = urlparse.urlsplit(self.registry_url)
+            registries = pull_secret['auths']
+            if parse.netloc in registries:
+                creds = base64.b64decode(registries[parse.netloc]['auth'])
+                user, passw = creds.split(':')
 
         # Use basic auth information for retrieving tokens from auth server and for downloading
         # with basic auth
-        self.auth_downloader = HTTPThreadedDownloader(copy.deepcopy(self.download_config),
-                                                      AggregatingEventListener())
+        dc = copy.deepcopy(self.download_config)
+        if user and passw:
+            dc.basic_auth_username = user
+            dc.basic_auth_password = passw
+        self.auth_downloader = HTTPThreadedDownloader(dc, AggregatingEventListener())
         self.download_config.basic_auth_username = None
         self.download_config.basic_auth_password = None
         self.downloader = HTTPThreadedDownloader(self.download_config, AggregatingEventListener())

plugins/test/unit/plugins/test_registry.py

@@ -373,14 +373,16 @@ def test___init__(self):
         download_config = DownloaderConfig(max_concurrent=25)
         registry_url = 'https://registry.example.com'
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
 
         self.assertEqual(r.name, name)
         self.assertEqual(r.download_config, download_config)
         self.assertEqual(r.registry_url, registry_url)
         self.assertEqual(type(r.downloader), HTTPThreadedDownloader)
         self.assertEqual(r.downloader.config, download_config)
         self.assertEqual(r.working_dir, working_dir)
+        self.assertEqual(r.repo_config, config)
 
     def test_api_version_check_incorrect_header(self):
         """
@@ -403,7 +405,8 @@ def download_one(request):
         download_config = DownloaderConfig(max_concurrent=25)
         registry_url = 'https://registry.example.com'
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
         r.downloader.download_one = mock.MagicMock(side_effect=download_one)
 
         self.assertFalse(r.api_version_check())
@@ -417,7 +420,8 @@ def test_api_version_check_ioerror(self, mock_get_path):
         download_config = DownloaderConfig(max_concurrent=25)
         registry_url = 'https://registry.example.com'
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
 
         self.assertFalse(r.api_version_check())
 
@@ -444,7 +448,8 @@ def download_one(request):
         download_config = DownloaderConfig(max_concurrent=25)
         registry_url = 'https://registry.example.com'
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
         r.downloader.download_one = mock.MagicMock(side_effect=download_one)
 
         # This should not raise an Exception
@@ -470,7 +475,8 @@ def download_one(request):
         download_config = DownloaderConfig(max_concurrent=25)
         registry_url = 'https://registry.example.com'
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
         r.downloader.download_one = mock.MagicMock(side_effect=download_one)
 
         # This should not raise an Exception
@@ -493,7 +499,8 @@ def test_create_blob_download_request(self):
         download_config = DownloaderConfig(max_concurrent=25)
         registry_url = 'https://registry.example.com'
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
         digest = 'sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef'
 
         request = r.create_blob_download_request(digest)
@@ -526,7 +533,8 @@ def download_one(request):
         download_config = DownloaderConfig(max_concurrent=25)
         registry_url = 'https://registry.example.com'
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
         r.downloader.download_one = mock.MagicMock(side_effect=download_one)
         digest = 'sha256:46356a7d9575b4cee21e7867b1b83a51788610b7719a616096d943b44737ad9a'
         with open(os.path.join(TEST_DATA_PATH, 'manifest_repeated_layers.json')) as manifest_file:
@@ -557,7 +565,8 @@ def download_one(request):
         download_config = DownloaderConfig(max_concurrent=25)
         registry_url = 'https://registry.example.com'
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
         r.downloader.download_one = mock.MagicMock(side_effect=download_one)
 
         tags = r.get_tags()
@@ -573,7 +582,8 @@ def test_get_tags_failed(self, mock_download_one):
         download_config = DownloaderConfig()
         registry_url = 'https://registry.example.com'
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
 
         with self.assertRaises(PulpCodedException) as assertion:
             r.get_tags()
@@ -590,7 +600,8 @@ def test__get_path_failed(self, mock_download_one, mock_request_token):
         download_config = DownloaderConfig(max_concurrent=25)
         registry_url = 'https://registry.example.com'
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
 
         report = DownloadReport(registry_url + '/some/path', StringIO())
         report.error_report['response_code'] = httplib.UNAUTHORIZED
@@ -621,7 +632,8 @@ def download_one(request):
         download_config = DownloaderConfig(max_concurrent=25)
         registry_url = 'https://registry.example.com'
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
         r.downloader.download_one = mock.MagicMock(side_effect=download_one)
 
         headers, body = r._get_path('/some/path')
@@ -666,7 +678,8 @@ def test_dockerhub_v2_registry_without_namespace(self, http_threaded_downloader)
         registry_url = "https://registry-1.docker.io"
         download_config = mock.MagicMock()
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
         self.assertEqual('library/test_image', r.name, "Non-name-spaced image not prepended")
 
     @mock.patch('pulp_docker.plugins.registry.HTTPThreadedDownloader')
@@ -675,7 +688,8 @@ def test_dockerhub_v2_registry_with_namespace(self, http_threaded_downloader):
         registry_url = "https://registry-1.docker.io"
         download_config = mock.MagicMock()
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
         self.assertNotEqual('library/library/test_image', r.name,
                             "Name-spaced image prepended with library")
 
@@ -685,7 +699,8 @@ def test_non_dockerhub_v2_registry_with_namespace(self, http_threaded_downloader
         registry_url = "https://somewhere.not-docker.io"
         download_config = mock.MagicMock()
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
         self.assertNotEqual('library/library/test_image', r.name,
                             "Name-spaced Non-docker hub image prepended with library")
 
@@ -695,6 +710,7 @@ def test_non_dockerhub_v2_registry_without_namespace(self, http_threaded_downloa
         registry_url = "https://somewhere.not-docker.io"
         download_config = mock.MagicMock()
         working_dir = '/a/working/dir'
-        r = registry.V2Repository(name, download_config, registry_url, working_dir)
+        config = {}
+        r = registry.V2Repository(name, download_config, registry_url, working_dir, config)
         self.assertNotEqual('library/test_image', r.name,
                             "Non-docker hub image prepended with library")