Merge pull request #14 from scrapinghub/sc737

Use apikey as username by default

VERSION

@@ -1 +1 @@
-0.1.1
+0.2.2

docs/deploy-custom-image.rst

@@ -299,6 +299,16 @@ Set the password to authenticate in the Docker registry.
 Set the email to authenticate in the Docker registry (if needed).
 
 
+.. function:: --apikey <text>
+
+(beta) Use provided apikey to authenticate in the Scrapy Cloud Docker registry.
+
+
+.. function:: --insecure
+
+Use the Docker registry in insecure mode.
+
+
 .. function:: -d/--debug
 
 Increase the tool's verbosity.
@@ -363,6 +373,16 @@ Set the password to authenticate in the registry.
 Set the email to authenticate in the Docker registry (if needed).
 
 
+.. function:: --apikey <text>
+
+(beta) Use provided apikey to authenticate in the Scrapy Cloud Docker registry.
+
+
+.. function:: --insecure
+
+Use the Docker registry in insecure mode.
+
+
 .. function:: --async
 
 Make deploy asynchronous. When enabled, the tool will exit as soon as the deploy is started in background. You can then check the status of your deploy task periodically via the :ref:`check <commands-check>` command.

requirements.txt

@@ -13,6 +13,6 @@ ruamel.base==1.0.0        # via ruamel.yaml
 ruamel.ordereddict==0.4.9  # via ruamel.yaml
 ruamel.yaml==0.10.20      # via shub
 scrapinghub==1.7.0        # via shub
-shub==2.0.0
+shub==2.2.0
 six==1.10.0
 websocket-client==0.35.0  # via docker-py

setup.py

@@ -16,7 +16,13 @@
     },
     include_package_data=True,
     zip_safe=False,
-    install_requires=['click', 'requests', 'six', 'shub', 'docker-py'],
+    install_requires=[
+        'click',
+        'requests',
+        'six',
+        'shub>=0.2.2',
+        'docker-py',
+    ],
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',

shub_image/deploy.py

@@ -48,18 +48,27 @@
 @click.option("--username", help="docker registry name")
 @click.option("--password", help="docker registry password")
 @click.option("--email", help="docker registry email")
+@click.option("--apikey", help="SH apikey to use built-in registry")
+@click.option("--insecure", is_flag=True, help="use insecure registry")
 @click.option("--async", is_flag=True, help="enable asynchronous mode")
-def cli(target, debug, version, username, password, email, async):
-    deploy_cmd(target, version, username, password, email, async)
+def cli(target, debug, version, username, password, email,
+        apikey, insecure, async):
+    deploy_cmd(target, version, username, password, email,
+        apikey, insecure, async)
 
 
-def deploy_cmd(target, version, username, password, email, async):
+def deploy_cmd(target, version, username, password, email,
+        apikey, insecure, async):
     config = utils.load_release_config()
-    project, endpoint, apikey = config.get_target(target)
+    project, endpoint, target_apikey = config.get_target(target)
     image = config.get_image(target)
     version = version or config.get_version()
     image_name = utils.format_image_name(image, version)
+    username, password = utils.get_credentials(
+        username=username, password=password, insecure=insecure,
+        apikey=apikey, target_apikey=target_apikey)
 
+    apikey = apikey or target_apikey
     params = _prepare_deploy_params(
         project, version, image_name, endpoint, apikey,
         username, password, email)

shub_image/push.py

@@ -30,14 +30,20 @@
 @click.option("--username", help="docker registry name")
 @click.option("--password", help="docker registry password")
 @click.option("--email", help="docker registry email")
-def cli(target, debug, version, username, password, email):
-    push_cmd(target, version, username, password, email)
+@click.option("--apikey", help="SH apikey to use built-in registry")
+@click.option("--insecure", is_flag=True, help="use insecure registry")
+def cli(target, debug, version, username, password, email, apikey, insecure):
+    push_cmd(target, version, username, password, email, apikey, insecure)
 
 
-def push_cmd(target, version, username, password, email):
+def push_cmd(target, version, username, password, email, apikey, insecure):
     client = utils.get_docker_client()
     config = utils.load_release_config()
     image = config.get_image(target)
+    username, password = utils.get_credentials(
+        username=username, password=password, insecure=insecure,
+        apikey=apikey, target_apikey=config.get_apikey(target))
+
     if username:
         _execute_push_login(client, image, username, password, email)
     image_name = utils.format_image_name(image, version)
@@ -60,10 +66,6 @@ def _execute_push_login(client, image, username, password, email):
     """Login if there're provided credentials for the registry"""
     components = image.split('/')
     registry = components[0] if len(components) == 3 else None
-    if password is None:
-        # Missing password leads to auth request to registry authentication service
-        # without 'account' query parameter which breaks login procedure.
-        password = ' '
     resp = client.login(username=username, password=password,
                         email=email, registry=registry, reauth=False)
     if not (isinstance(resp, dict) and 'username' in resp or

shub_image/upload.py

@@ -25,8 +25,12 @@
 @click.option("--username", help="docker registry name")
 @click.option("--password", help="docker registry password")
 @click.option("--email", help="docker registry email")
+@click.option("--apikey", help="SH apikey to use built-in registry")
+@click.option("--insecure", is_flag=True, help="use insecure registry")
 @click.option("--async", is_flag=True, help="enable asynchronous mode")
-def cli(target, debug, version, username, password, email, async):
+def cli(target, debug, version, username, password, email,
+        apikey, insecure, async):
     build.build_cmd(target, version)
-    push.push_cmd(target, version, username, password, email)
-    deploy.deploy_cmd(target, version, username, password, email, async)
+    push.push_cmd(target, version, username, password, email, apikey, insecure)
+    deploy.deploy_cmd(target, version, username, password, email,
+                      apikey, insecure, async)

shub_image/utils.py

@@ -136,6 +136,31 @@ def format_image_name(image_name, image_tag):
     return '{}:{}'.format(image_name, image_tag)
 
 
+def get_credentials(username=None, password=None, insecure=False,
+                    apikey=None, target_apikey=None):
+    """ A helper function to get credentials based on cmdline options.
+
+    Returns a tuple with 2 strings: (username, password).
+
+    When working with registries where only username matters:
+    missing password leads to auth request to registry authentication service
+    without 'account' query parameter which breaks login.
+    """
+    if insecure:
+        return None, None
+    elif apikey:
+        return apikey, ' '
+    elif username:
+        if password is None:
+            raise click.BadParameter(
+                'Password is required when passing username.')
+        return username, password
+    elif password:
+        raise click.BadParameter(
+            'Username is required when passing password.')
+    return target_apikey, ' '
+
+
 def store_status_url(status_url, limit):
     """Load status file and update it with a url"""
     data = _load_status_file(STATUS_FILE_LOCATION)

tests/test_deploy.py

@@ -32,11 +32,42 @@ def test_cli(self, list_mocked, post_mocked, get_mocked):
             runner = CliRunner()
             result = runner.invoke(cli, ["dev", "--version", "test"])
             assert result.exit_code == 0
+            auth_cfg = '{"email": null, "password": " ", "username": "abcdef"}'
             post_mocked.assert_called_with(
-                'https://dash.scrapinghub.com/api/releases/deploy.json',
+                'https://app.scrapinghub.com/api/releases/deploy.json',
                 allow_redirects=False, auth=('abcdef', ''),
-                data={'project': 12345, 'pull_insecure_registry': True,
+                data={'project': 12345,
                       'version': u'test',
+                      'pull_auth_config': auth_cfg,
+                      'image_url': 'registry/user/project:test',
+                      'spiders': 'a1f,abc,spi-der',
+                      'scripts': 'scriptA.py,scriptB.py'}, timeout=300)
+            get_mocked.assert_called_with('https://status-url', timeout=300)
+
+    @mock.patch('requests.get')
+    @mock.patch('requests.post')
+    @mock.patch('shub_image.list.list_cmd')
+    def test_cli_insecure_registry(self, list_mocked, post_mocked, get_mocked):
+        list_mocked.return_value = ['a1f', 'abc', 'spi-der']
+        post_req = mock.Mock()
+        post_req.headers = {'location': 'https://status-url'}
+        post_mocked.return_value = post_req
+
+        with FakeProjectDirectory() as tmpdir:
+            add_scrapy_fake_config(tmpdir)
+            add_sh_fake_config(tmpdir)
+            add_fake_setup_py(tmpdir)
+
+            runner = CliRunner()
+            result = runner.invoke(
+                cli, ["dev", "--version", "test", "--insecure"])
+            assert result.exit_code == 0
+            post_mocked.assert_called_with(
+                'https://app.scrapinghub.com/api/releases/deploy.json',
+                allow_redirects=False, auth=('abcdef', ''),
+                data={'project': 12345,
+                      'version': u'test',
+                      'pull_insecure_registry': True,
                       'image_url': 'registry/user/project:test',
                       'spiders': 'a1f,abc,spi-der',
                       'scripts': 'scriptA.py,scriptB.py'}, timeout=300)

tests/test_list.py

@@ -77,7 +77,7 @@ def test_cli(self, get_mocked, get_client_mock):
             assert result.exit_code == 0
             assert result.output.endswith('abc\ndef\ndsd\n')
         get_mocked.assert_called_with(
-            'https://dash.scrapinghub.com/api/settings/get.json',
+            'https://app.scrapinghub.com/api/settings/get.json',
             allow_redirects=False, auth=('abcdef', ''),
             params={'project': 12345}, timeout=300)
 

tests/test_push.py

@@ -11,8 +11,9 @@
 @mock.patch('shub_image.utils.get_docker_client')
 class TestPushCli(TestCase):
 
-    def test_cli(self, mocked_method):
+    def test_cli_with_apikey_login(self, mocked_method):
         mocked = mock.MagicMock()
+        mocked.login.return_value = {"Status": "Login Succeeded"}
         mocked.push.return_value = [
             '{"stream":"In process"}',
             '{"status":"Successfully pushed"}']
@@ -24,9 +25,9 @@ def test_cli(self, mocked_method):
             assert result.exit_code == 0
             mocked.push.assert_called_with(
                 'registry/user/project:test',
-                insecure_registry=True, stream=True)
+                insecure_registry=False, stream=True)
 
-    def test_cli_with_login(self, mocked_method):
+    def test_cli_with_custom_login(self, mocked_method):
         mocked = mock.MagicMock()
         mocked.login.return_value = {"Status": "Login Succeeded"}
         mocked.push.return_value = [
@@ -47,6 +48,24 @@ def test_cli_with_login(self, mocked_method):
                 'registry/user/project:test',
                 insecure_registry=False, stream=True)
 
+    def test_cli_with_insecure_registry(self, mocked_method):
+        mocked = mock.MagicMock()
+        mocked.login.return_value = {"Status": "Login Succeeded"}
+        mocked.push.return_value = [
+            '{"stream":"In process"}',
+            '{"status":"Successfully pushed"}']
+        mocked_method.return_value = mocked
+        with FakeProjectDirectory() as tmpdir:
+            add_sh_fake_config(tmpdir)
+            runner = CliRunner()
+            result = runner.invoke(
+                cli, ["dev", "--version", "test", "--insecure"])
+            assert result.exit_code == 0
+            assert not mocked.login.called
+            mocked.push.assert_called_with(
+                'registry/user/project:test',
+                insecure_registry=True, stream=True)
+
     def test_cli_with_login_username_only(self, mocked_method):
         mocked = mock.MagicMock()
         mocked.login.return_value = {"Status": "Login Succeeded"}
@@ -58,7 +77,7 @@ def test_cli_with_login_username_only(self, mocked_method):
             add_sh_fake_config(tmpdir)
             runner = CliRunner()
             result = runner.invoke(
-                cli, ["dev", "--version", "test", "--username", "apikey"])
+                cli, ["dev", "--version", "test", "--apikey", "apikey"])
             assert result.exit_code == 0
             mocked.login.assert_called_with(
                 email=None, password=' ',

tests/test_upload.py

@@ -17,8 +17,10 @@ def test_cli(self, build, push, deploy):
         result = runner.invoke(
             cli, ["dev", "-d", "--version", "test",
                   "--username", "user", "--password", "pass",
-                  "--email", "mail", "--async"])
+                  "--email", "mail", "--async", "--apikey", "apikey"])
         assert result.exit_code == 0
         build.assert_called_with('dev', 'test')
-        push.assert_called_with('dev', 'test', 'user', 'pass', 'mail')
-        deploy.assert_called_with('dev', 'test', 'user', 'pass', 'mail', True)
+        push.assert_called_with(
+            'dev', 'test', 'user', 'pass', 'mail', "apikey", False)
+        deploy.assert_called_with(
+            'dev', 'test', 'user', 'pass', 'mail', "apikey", False, True)

tests/test_utils.py

@@ -1,18 +1,21 @@
-import json
 import os
 import sys
+import json
+import tempfile
+from unittest import TestCase
 
-import docker
 import mock
+import click
+import docker
+import pytest
 import StringIO
-import tempfile
-from unittest import TestCase
 from shub import exceptions as shub_exceptions
 
 from shub_image.utils import missing_modules
 from shub_image.utils import get_project_dir
 from shub_image.utils import get_docker_client
 from shub_image.utils import format_image_name
+from shub_image.utils import get_credentials
 
 from shub_image.utils import ReleaseConfig
 from shub_image.utils import load_release_config
@@ -106,6 +109,15 @@ def test_custom_docker_client_workaround(self):
         assert json.loads(result[0])['id'] == '2.7'
         assert json.loads(result[1])['id'] == '5c90d4a2d1a8'
 
+    def test_get_credentials(self):
+        assert get_credentials(insecure=True) == (None, None)
+        assert get_credentials(apikey='apikey') == ('apikey', ' ')
+        assert get_credentials(
+            username='user', password='pass') == ('user', 'pass')
+        with pytest.raises(click.BadParameter):
+            get_credentials(username='user')
+        assert get_credentials(target_apikey='tapikey') == ('tapikey', ' ')
+
 
 class ReleaseConfigTest(TestCase):
 
@@ -125,7 +137,7 @@ def test_load(self):
         config.load(stream)
         assert getattr(config, 'projects') == {'dev': 123, 'prod': 321}
         assert getattr(config, 'endpoints') == {
-            'default': 'https://dash.scrapinghub.com/api/',
+            'default': 'https://app.scrapinghub.com/api/',
             'dev': 'http://127.0.0.1/api/scrapyd/'}
         assert config.images == {
             'dev': 'registry/user/project',