diff --git a/.travis.yml b/.travis.yml index b5f5611..e0813ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,18 @@ language: python -python: 2.7 +python: 3.5 branches: only: - master - /^\d\.\d+$/ - /^v\d\.\d+\.\d+(rc\d+|dev\d+)?$/ env: -- TOXENV=py27 +- TOX_ENV=py27 +- TOX_ENV=py33 +- TOX_ENV=py34 +- TOX_ENV=py35 install: - pip install -U tox twine wheel codecov -script: tox +script: tox -e $TOX_ENV after_success: - codecov cache: diff --git a/docs/conf.py b/docs/conf.py index cf0d706..a5ca137 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,9 +12,7 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os -import shlex from os import path from datetime import datetime @@ -26,12 +24,13 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# import sys +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -47,7 +46,7 @@ source_suffix = '.rst' # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' @@ -75,9 +74,9 @@ # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -85,27 +84,27 @@ # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -120,26 +119,26 @@ # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -149,62 +148,62 @@ # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. htmlhelp_basename = 'shub-imagedoc' @@ -212,17 +211,17 @@ # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', -# Latex figure (float) alignment -#'figure_align': 'htbp', + # Latex figure (float) alignment + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples @@ -235,23 +234,23 @@ # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- @@ -264,7 +263,7 @@ ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -279,19 +278,23 @@ ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False -### Following is taken from https://github.com/snide/sphinx_rtd_theme#using-this-theme-locally-then-building-on-read-the-docs -# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org + +# Following is taken from https://github.com/snide/sphinx_rtd_theme# +# using-this-theme-locally-then-building-on-read-the-docs + +# on_rtd is whether we are on readthedocs.org, +# this line of code grabbed from docs.readthedocs.org on_rtd = os.environ.get('READTHEDOCS', None) == 'True' if not on_rtd: # only import and set the theme if we're building docs locally @@ -299,5 +302,4 @@ html_theme = 'sphinx_rtd_theme' html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -# otherwise, readthedocs.org uses their theme by default, so no need to specify it -### end +# otherwise, readthedocs.org uses their theme by default, no need to specify it diff --git a/requirements.in b/requirements.in index 7819a23..34d6424 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,6 @@ click docker-py +PyYAML requests retrying shub diff --git a/requirements.txt b/requirements.txt index d71e528..84ec6f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,24 @@ # # This file is autogenerated by pip-compile -# Make changes in requirements.in, then run this to update: +# To update, run: # -# pip-compile requirements.in +# pip-compile --output-file requirements.txt requirements.in # -click==6.2 -docker-py==1.7.0 -hubstorage==0.22.0 # via shub -requests==2.9.1 -retrying==1.3.3 # via hubstorage -ruamel.base==1.0.0 # via ruamel.yaml -ruamel.ordereddict==0.4.9 # via ruamel.yaml -ruamel.yaml==0.10.20 # via shub + +backports.ssl-match-hostname==3.5.0.1 # via docker-py +click==6.6 +docker-py==1.10.6 +docker-pycreds==0.2.1 # via docker-py +hubstorage==0.23.5 # via shub +ipaddress==1.0.17 # via docker-py +PyYAML==3.12 # via shub +requests==2.10.0 +retrying==1.3.3 scrapinghub==1.7.0 # via shub -shub==2.2.0 +shub==2.4.2 six==1.10.0 -websocket-client==0.35.0 # via docker-py +websocket-client==0.37.0 # via docker-py + +# The following packages are commented out because they are +# considered to be unsafe in a requirements file: +# pip # via shub diff --git a/setup.py b/setup.py index fd12614..b514deb 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ 'requests', 'six', 'shub>=0.2.5', - 'docker-py', + 'docker-py>=1.10.0', ], classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/shub_image/build.py b/shub_image/build.py index 311a69d..eb6b8f4 100644 --- a/shub_image/build.py +++ b/shub_image/build.py @@ -1,6 +1,5 @@ import os import re -import json import click import warnings @@ -48,8 +47,7 @@ def build_cmd(target, version): raise shub_exceptions.BadParameterException( 'Dockerfile is not found, please use shub-image init cmd') is_built = False - for line in client.build(path=project_dir, tag=image_name): - data = json.loads(line) + for data in client.build(path=project_dir, tag=image_name, decode=True): if 'stream' in data: utils.debug_log("{}".format(data['stream'][:-1])) is_built = re.search( diff --git a/shub_image/check.py b/shub_image/check.py index 269fd75..db498b6 100644 --- a/shub_image/check.py +++ b/shub_image/check.py @@ -1,4 +1,3 @@ -import os import click import requests diff --git a/shub_image/deploy.py b/shub_image/deploy.py index f5f773a..d959a71 100644 --- a/shub_image/deploy.py +++ b/shub_image/deploy.py @@ -3,12 +3,12 @@ import ast import json import time +import textwrap + import click import requests -import textwrap -import subprocess -from urlparse import urljoin from retrying import retry +from six.moves.urllib.parse import urljoin from shub.deploy import list_targets from shub.exceptions import ShubException @@ -55,11 +55,11 @@ def cli(target, debug, version, username, password, email, apikey, insecure, async): deploy_cmd(target, version, username, password, email, - apikey, insecure, async) + apikey, insecure, async) def deploy_cmd(target, version, username, password, email, - apikey, insecure, async): + apikey, insecure, async): config = utils.load_release_config() project, endpoint, target_apikey = config.get_target(target) image = config.get_image(target) diff --git a/shub_image/init.py b/shub_image/init.py index 603a980..a931713 100644 --- a/shub_image/init.py +++ b/shub_image/init.py @@ -1,8 +1,10 @@ import os -import click import textwrap from string import Template +import click +from six.moves import input + from shub import exceptions as shub_exceptions from shub import utils as shub_utils from shub_image import utils @@ -43,7 +45,8 @@ System deps for Dockerfile: By default there're several system deps to be included to the Dockerfile ({}), -you can extend it via --add-deps option, or redefine at all with --base-deps option. +you can extend it via --add-deps option, or redefine at all with --base-deps +option. Python deps for Dockerfile: The correct way to install python deps is using requirements.txt. If there's @@ -104,7 +107,7 @@ def cli(project, base_image, base_deps, add_deps, requirements): "no": False, "n": False} while True: dockefile_path = os.path.join(project_dir, 'Dockerfile') - choice = raw_input("Save to {}: (y/n)".format(dockefile_path)).lower() + choice = input("Save to {}: (y/n)".format(dockefile_path)).lower() if choice in valid: if valid[choice]: with open(dockefile_path, 'w') as dockerfile: diff --git a/shub_image/list.py b/shub_image/list.py index 0bdddb8..4f39d26 100644 --- a/shub_image/list.py +++ b/shub_image/list.py @@ -1,9 +1,8 @@ -import os -import re import json + import click import requests -from urlparse import urljoin +from six.moves.urllib.parse import urljoin from shub import exceptions as shub_exceptions from shub.deploy import list_targets diff --git a/shub_image/push.py b/shub_image/push.py index 797a2ad..c51bee9 100644 --- a/shub_image/push.py +++ b/shub_image/push.py @@ -1,4 +1,3 @@ -import json import click from shub.deploy import list_targets @@ -48,9 +47,8 @@ def push_cmd(target, version, username, password, email, apikey, insecure): _execute_push_login(client, image, username, password, email) image_name = utils.format_image_name(image, version) click.echo("Pushing {} to the registry.".format(image_name)) - for line in client.push(image_name, stream=True, + for data in client.push(image_name, stream=True, decode=True, insecure_registry=not bool(username)): - data = json.loads(line) if 'status' in data: utils.debug_log("Logs:{} {}".format(data['status'], data.get('progress'))) diff --git a/shub_image/tool.py b/shub_image/tool.py index 6d407fd..742b93d 100644 --- a/shub_image/tool.py +++ b/shub_image/tool.py @@ -1,19 +1,6 @@ import click import importlib import shub_image -from shub_image.utils import missing_modules - - -def missingmod_cmd(modules): - modlist = ", ".join(modules) - - @click.command(help="*DISABLED* - requires %s" % modlist) - @click.pass_context - def cmd(ctx): - click.echo("Error: '%s' command requires %s" % - (ctx.info_name, modlist)) - ctx.exit(1) - return cmd @click.group(help="Scrapinghub release tool") @@ -22,22 +9,18 @@ def cli(): pass -module_deps = { - "init": [], - "build": ["docker"], - "list": ["docker"], - "test": ["docker"], - "push": ["docker"], - "deploy": ["docker"], - "upload": ["docker"], - "check": [], -} - -for command, modules in module_deps.items(): - m = missing_modules(*modules) - if m: - cli.add_command(missingmod_cmd(m), command) - else: - module_path = "shub_image." + command - command_module = importlib.import_module(module_path) - cli.add_command(command_module.cli, command) +module_deps = [ + "init", + "build", + "list", + "test", + "push", + "deploy", + "upload", + "check", +] + +for command in module_deps: + module_path = "shub_image." + command + command_module = importlib.import_module(module_path) + cli.add_command(command_module.cli, command) diff --git a/shub_image/utils.py b/shub_image/utils.py index 74fd39c..bbc2ac9 100644 --- a/shub_image/utils.py +++ b/shub_image/utils.py @@ -1,11 +1,9 @@ import os import re import click -import importlib import contextlib -from six import string_types -import ruamel.yaml as yaml +import yaml from shub import config as shub_config from shub import utils as shub_utils @@ -14,7 +12,7 @@ DEFAULT_DOCKER_VERSION = '1.17' STATUS_FILE_LOCATION = '.releases' -_VALIDSPIDERNAME = re.compile('^[a-z0-9][-._a-z0-9]+$', re.I) +_VALIDSPIDERNAME = re.compile(b'^[a-z0-9][-._a-z0-9]+$', re.I) DOCKER_UNAVAILABLE_MSG = """ Detected error connecting to Docker daemon's host. @@ -83,20 +81,6 @@ def load_release_config(): return shub_config.load_shub_config() -def missing_modules(*modules): - """Receives a list of module names and returns those which are missing""" - missing = [] - for module_name in modules: - try: - importlib.import_module(module_name) - except ImportError: - if module_name == 'docker': - missing.append('docker-py') - else: - missing.append(module_name) - return missing - - def get_project_dir(): """ A helper to get project root dir. Used by init/build command to locate Dockerfile. @@ -115,26 +99,13 @@ def get_docker_client(validate=True): except ImportError: raise ImportError('You need docker-py installed for the cmd') - class CustomDockerClient(docker.Client): - - # XXX: workaround for https://github.com/docker/docker-py/issues/1059 - def _stream_helper(self, response, decode=False): - it = super(CustomDockerClient, self)._stream_helper(response, decode=decode) - for data in it: - if not isinstance(data, string_types): - yield data - for line in data.split('\r\n'): - line = line.strip() - if line: - yield line - docker_host = os.environ.get('DOCKER_HOST') tls_config = None if os.environ.get('DOCKER_TLS_VERIFY', False): tls_cert_path = os.environ.get('DOCKER_CERT_PATH') if not tls_cert_path: tls_cert_path = os.path.join(os.path.expanduser('~'), '.docker') - apply_path_fun = lambda name: os.path.join(tls_cert_path, name) + apply_path_fun = lambda name: os.path.join(tls_cert_path, name) # noqa tls_config = docker.tls.TLSConfig( client_cert=(apply_path_fun('cert.pem'), apply_path_fun('key.pem')), @@ -142,9 +113,9 @@ def _stream_helper(self, response, decode=False): assert_hostname=False) docker_host = docker_host.replace('tcp://', 'https://') version = os.environ.get('DOCKER_VERSION', DEFAULT_DOCKER_VERSION) - client = CustomDockerClient(base_url=docker_host, - version=version, - tls=tls_config) + client = docker.Client(base_url=docker_host, + version=version, + tls=tls_config) if validate: validate_connection_with_docker_daemon(client) return client @@ -241,7 +212,7 @@ def _load_status_file(path): with open(path, 'r') as f: try: data = yaml.safe_load(f) - except yaml.YAMLError, exc: + except yaml.YAMLError as exc: raise shub_exceptions.BadConfigException( "Error reading releases file:\n{}".format(exc)) if not isinstance(data, dict): diff --git a/tests/test_build.py b/tests/test_build.py index e96e6c5..69a4c33 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -10,14 +10,15 @@ from .utils import add_fake_dockerfile from .utils import add_scrapy_fake_config + @mock.patch('shub_image.utils.get_docker_client') class TestBuildCli(TestCase): def test_cli(self, mocked_method): mocked = mock.MagicMock() mocked.build.return_value = [ - '{"stream":"all is ok"}', - '{"stream":"Successfully built 12345"}'] + {"stream": "all is ok"}, + {"stream": "Successfully built 12345"}] mocked_method.return_value = mocked with FakeProjectDirectory() as tmpdir: add_scrapy_fake_config(tmpdir) @@ -29,14 +30,14 @@ def test_cli(self, mocked_method): result = runner.invoke(cli, ["dev", "-d"]) assert result.exit_code == 0 mocked.build.assert_called_with( - path=tmpdir, tag='registry/user/project:1.0') + decode=True, path=tmpdir, tag='registry/user/project:1.0') assert os.path.isfile(setup_py_path) def test_cli_custom_version(self, mocked_method): mocked = mock.MagicMock() mocked.build.return_value = [ - '{"stream":"all is ok"}', - '{"stream":"Successfully built 12345"}'] + {"stream": "all is ok"}, + {"stream": "Successfully built 12345"}] mocked_method.return_value = mocked with FakeProjectDirectory() as tmpdir: add_scrapy_fake_config(tmpdir) @@ -46,13 +47,13 @@ def test_cli_custom_version(self, mocked_method): result = runner.invoke(cli, ["dev", "--version", "test"]) assert result.exit_code == 0 mocked.build.assert_called_with( - path=tmpdir, tag='registry/user/project:test') + decode=True, path=tmpdir, tag='registry/user/project:test') def test_cli_no_dockerfile(self, mocked_method): mocked = mock.MagicMock() mocked.build.return_value = [ - '{"error":"Minor","errorDetail":"Testing output"}', - '{"stream":"Successfully built 12345"}'] + {"error": "Minor", "errorDetail": "Testing output"}, + {"stream": "Successfully built 12345"}] mocked_method.return_value = mocked with FakeProjectDirectory() as tmpdir: add_scrapy_fake_config(tmpdir) @@ -64,7 +65,7 @@ def test_cli_no_dockerfile(self, mocked_method): def test_cli_fail(self, mocked_method): mocked = mock.MagicMock() - mocked.build.return_value = ['{"error":"Minor","errorDetail":"Test"}'] + mocked.build.return_value = [{"error": "Minor", "errorDetail": "Test"}] mocked_method.return_value = mocked with FakeProjectDirectory() as tmpdir: add_scrapy_fake_config(tmpdir) diff --git a/tests/test_check.py b/tests/test_check.py index d304b9b..14e7721 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -1,25 +1,27 @@ -import os import mock from click.testing import CliRunner from unittest import TestCase from shub import exceptions as shub_exceptions + from shub_image.check import cli from shub_image import utils - from .utils import FakeProjectDirectory + class TestCheckCli(TestCase): @mock.patch('requests.get') def test_cli(self, mocked): - with FakeProjectDirectory() as tmpdir: + # the test creates .releases file locally + # this context manager cleans it in the end + with FakeProjectDirectory(): runner = CliRunner() result = runner.invoke(cli, []) assert result.exit_code == \ shub_exceptions.NotFoundException.exit_code deploy_id1 = utils.store_status_url('http://linkA', 2) deploy_id2 = utils.store_status_url('http://linkB', 2) - deploy_id3 = utils.store_status_url('http://linkC', 2) + utils.store_status_url('http://linkC', 2) # get latest (deploy 3) result = runner.invoke(cli, []) diff --git a/tests/test_deploy.py b/tests/test_deploy.py index 3d4ba67..f44be70 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -1,4 +1,3 @@ -import os import mock from click.testing import CliRunner from unittest import TestCase diff --git a/tests/test_init.py b/tests/test_init.py index af7152c..8732e98 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -1,7 +1,6 @@ import os from click.testing import CliRunner from unittest import TestCase -from shub import exceptions as shub_exceptions from shub_image.init import cli from shub_image.init import _format_system_deps @@ -67,11 +66,11 @@ def test_wrap(self): assert _wrap(short_cmd) == short_cmd assert _wrap(short_cmd + ' ' + short_cmd) == ( short_cmd + ' ' + ' '.join(short_cmd.split()[:3]) + - " \\\n " + ' '.join(short_cmd.split()[3:])) + " \\\n " + ' '.join(short_cmd.split()[3:])) def test_format_system_deps(self): # no deps at all - assert _format_system_deps('-', None) == None + assert _format_system_deps('-', None) is None # base deps only assert _format_system_deps('a,b,cd', None) == ( "RUN apt-get update -qq && \\\n" diff --git a/tests/test_list.py b/tests/test_list.py index b0f8a1a..1434798 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -1,11 +1,9 @@ -import os import mock from click.testing import CliRunner from unittest import TestCase -from shub_image.list import cli, list_cmd +from shub_image.list import cli from shub_image.list import _run_list_cmd -from shub_image.list import _get_project_settings from .utils import FakeProjectDirectory from .utils import add_sh_fake_config @@ -14,11 +12,10 @@ class TestListCli(TestCase): def test_cli_no_sh_config(self): - with FakeProjectDirectory() as tmpdir: - runner = CliRunner() - result = runner.invoke(cli, ["dev", "-d", "--version", "test"]) - assert result.exit_code == 69 - assert 'Could not find image for dev' in result.output + runner = CliRunner() + result = runner.invoke(cli, ["dev", "-d", "--version", "test"]) + assert result.exit_code == 69 + assert 'Could not find image for dev' in result.output @mock.patch('shub_image.utils.get_docker_client') @mock.patch('requests.get') @@ -26,7 +23,7 @@ def test_cli_no_project(self, get_mocked, get_client_mock): client_mock = mock.Mock() client_mock.create_container.return_value = {'Id': '1234'} client_mock.wait.return_value = 0 - client_mock.logs.return_value = 'abc\ndef' + client_mock.logs.return_value = b'abc\ndef' get_client_mock.return_value = client_mock with FakeProjectDirectory() as tmpdir: @@ -62,7 +59,7 @@ def test_cli(self, get_mocked, get_client_mock): client_mock = mock.Mock() client_mock.create_container.return_value = {'Id': '1234'} client_mock.wait.return_value = 0 - client_mock.logs.return_value = 'abc\ndef\ndsd' + client_mock.logs.return_value = b'abc\ndef\ndsd' get_client_mock.return_value = client_mock get_settings_mock = mock.Mock() @@ -83,7 +80,7 @@ def test_cli(self, get_mocked, get_client_mock): class TestListCmd(TestCase): - + @mock.patch('shub_image.utils.get_docker_client') def test_run_list_cmd(self, get_client_mock): client_mock = mock.Mock() diff --git a/tests/test_push.py b/tests/test_push.py index e270756..f7a9f52 100644 --- a/tests/test_push.py +++ b/tests/test_push.py @@ -1,4 +1,3 @@ -import os import mock from click.testing import CliRunner from unittest import TestCase @@ -8,6 +7,7 @@ from .utils import FakeProjectDirectory from .utils import add_sh_fake_config + @mock.patch('shub_image.utils.get_docker_client') class TestPushCli(TestCase): @@ -15,8 +15,8 @@ def test_cli_with_apikey_login(self, mocked_method): mocked = mock.MagicMock() mocked.login.return_value = {"Status": "Login Succeeded"} mocked.push.return_value = [ - '{"stream":"In process"}', - '{"status":"Successfully pushed"}'] + {"stream": "In process"}, + {"status": "Successfully pushed"}] mocked_method.return_value = mocked with FakeProjectDirectory() as tmpdir: add_sh_fake_config(tmpdir) @@ -24,15 +24,15 @@ def test_cli_with_apikey_login(self, mocked_method): result = runner.invoke(cli, ["dev", "--version", "test"]) assert result.exit_code == 0 mocked.push.assert_called_with( - 'registry/user/project:test', + 'registry/user/project:test', decode=True, insecure_registry=False, stream=True) def test_cli_with_custom_login(self, mocked_method): mocked = mock.MagicMock() mocked.login.return_value = {"Status": "Login Succeeded"} mocked.push.return_value = [ - '{"stream":"In process"}', - '{"status":"Successfully pushed"}'] + {"stream": "In process"}, + {"status": "Successfully pushed"}] mocked_method.return_value = mocked with FakeProjectDirectory() as tmpdir: add_sh_fake_config(tmpdir) @@ -45,15 +45,15 @@ def test_cli_with_custom_login(self, mocked_method): email=u'mail', password=u'pass', reauth=False, registry='registry', username=u'user') mocked.push.assert_called_with( - 'registry/user/project:test', + 'registry/user/project:test', decode=True, insecure_registry=False, stream=True) def test_cli_with_insecure_registry(self, mocked_method): mocked = mock.MagicMock() mocked.login.return_value = {"Status": "Login Succeeded"} mocked.push.return_value = [ - '{"stream":"In process"}', - '{"status":"Successfully pushed"}'] + {"stream": "In process"}, + {"status": "Successfully pushed"}] mocked_method.return_value = mocked with FakeProjectDirectory() as tmpdir: add_sh_fake_config(tmpdir) @@ -63,15 +63,15 @@ def test_cli_with_insecure_registry(self, mocked_method): assert result.exit_code == 0 assert not mocked.login.called mocked.push.assert_called_with( - 'registry/user/project:test', + 'registry/user/project:test', decode=True, insecure_registry=True, stream=True) def test_cli_with_login_username_only(self, mocked_method): mocked = mock.MagicMock() mocked.login.return_value = {"Status": "Login Succeeded"} mocked.push.return_value = [ - '{"stream":"In process"}', - '{"status":"Successfully pushed"}'] + {"stream": "In process"}, + {"status": "Successfully pushed"}] mocked_method.return_value = mocked with FakeProjectDirectory() as tmpdir: add_sh_fake_config(tmpdir) @@ -83,7 +83,7 @@ def test_cli_with_login_username_only(self, mocked_method): email=None, password=' ', reauth=False, registry='registry', username='apikey') mocked.push.assert_called_with( - 'registry/user/project:test', + 'registry/user/project:test', decode=True, insecure_registry=False, stream=True) def test_cli_login_fail(self, mocked_method): @@ -102,7 +102,7 @@ def test_cli_login_fail(self, mocked_method): def test_cli_push_fail(self, mocked_method): mocked = mock.MagicMock() mocked.login.return_value = {"Status": "Login Succeeded"} - mocked.push.return_value = ['{"error":"Failed:(","errorDetail":""}'] + mocked.push.return_value = [{"error": "Failed:(", "errorDetail": ""}] mocked_method.return_value = mocked with FakeProjectDirectory() as tmpdir: add_sh_fake_config(tmpdir) diff --git a/tests/test_test.py b/tests/test_test.py index 8bc9b51..1ebe1bd 100644 --- a/tests/test_test.py +++ b/tests/test_test.py @@ -1,8 +1,6 @@ -import os -import sys import mock +import pytest from click.testing import CliRunner -from unittest import TestCase from shub import exceptions as shub_exceptions from shub_image.test import cli @@ -19,91 +17,74 @@ class MockedNotFound(Exception): """Mocking docker.errors.NotFound""" -def _mock_docker_errors_module(): - """ A helper to avoid using docker library at all""" - orig_import = __import__ - errors_mock = mock.Mock() - errors_mock.NotFound = MockedNotFound - def import_mock(name, *args): - if name == 'docker.errors': - return errors_mock - return orig_import(name, *args) - return import_mock - - -class TestTestCli(TestCase): - - @mock.patch('shub_image.utils.get_docker_client') - def test_cli(self, mocked_method): - """ This test mocks docker library to test the function itself """ - client = mock.Mock() - # mainly for several checks on status & logs - client.create_container.return_value = {'Id': '12345'} - client.wait.return_value = 0 - client.logs.return_value = 'some-logs' - mocked_method.return_value = client - # patching built-in import to use fake docker.errors - import_mock = _mock_docker_errors_module() - with mock.patch('__builtin__.__import__', side_effect=import_mock): - with FakeProjectDirectory() as tmpdir: - add_sh_fake_config(tmpdir) - runner = CliRunner() - result = runner.invoke( - cli, ["dev", "-d", "--version", "test"]) - assert result.exit_code == 0 - - -class TestTestTools(TestCase): - - def setUp(self): - self.client = mock.Mock() - self.client.create_container.return_value = {'Id': '12345'} - self.client.wait.return_value = 0 - self.client.logs.return_value = 'some-logs' - - def test_check_image_exists(self): - """ This test mocks docker library to test the function itself """ - # patching built-in import to use fake docker.errors - import_mock = _mock_docker_errors_module() - with mock.patch('__builtin__.__import__', side_effect=import_mock): - assert _check_image_exists('img', self.client) == None - self.client.inspect_image.side_effect = MockedNotFound() - self.assertRaises(shub_exceptions.NotFoundException, - _check_image_exists, 'image', self.client) - - def test_check_sh_entrypoint(self): - assert _check_sh_entrypoint('image', self.client) == None - self.client.create_container.assert_called_with( - image='image', - command=['pip', 'show', 'scrapinghub-entrypoint-scrapy']) - self.client.wait.return_value = 1 - self.assertRaises(shub_exceptions.NotFoundException, - _check_sh_entrypoint, 'image', self.client) - self.client.wait.return_value = 0 - self.client.logs.return_value = '' - self.assertRaises(shub_exceptions.NotFoundException, - _check_sh_entrypoint, 'image', self.client) - - def test_start_crawl(self): - assert _check_start_crawl_entry('image', self.client) == None - self.client.create_container.assert_called_with( - image='image', command=['which', 'start-crawl']) - self.client.wait.return_value = 1 - self.assertRaises(shub_exceptions.NotFoundException, - _check_start_crawl_entry, 'image', self.client) - self.client.wait.return_value = 0 - self.client.logs.return_value = '' - self.assertRaises(shub_exceptions.NotFoundException, - _check_start_crawl_entry, 'image', self.client) - - def test_run_docker_command(self): - assert _run_docker_command( - self.client, 'image-name', ['some', 'cmd']) == \ - (0, 'some-logs') - self.client.create_container.assert_called_with( - image='image-name', command=['some', 'cmd']) - self.client.start.assert_called_with({'Id': '12345'}) - self.client.wait.assert_called_with(container='12345') - self.client.logs.assert_called_with( - container='12345', stdout=True, stderr=False, - stream=False, timestamps=False) +@pytest.fixture +def docker_client(): + client = mock.Mock() + client.create_container.return_value = {'Id': '12345'} + client.wait.return_value = 0 + client.logs.return_value = 'some-logs' + return client + + +def test_test_cli(monkeypatch, docker_client): + """ This test mocks docker library to test the function itself """ + monkeypatch.setattr('docker.errors.NotFound', MockedNotFound) + monkeypatch.setattr('shub_image.utils.get_docker_client', + lambda *args, **kwargs: docker_client) + with FakeProjectDirectory() as tmpdir: + add_sh_fake_config(tmpdir) + runner = CliRunner() + result = runner.invoke( + cli, ["dev", "-d", "--version", "test"]) + assert result.exit_code == 0 + + +def test_check_image_exists(monkeypatch, docker_client): + assert _check_image_exists('img', docker_client) is None + + monkeypatch.setattr('docker.errors.NotFound', MockedNotFound) + docker_client.inspect_image.side_effect = MockedNotFound + with pytest.raises(shub_exceptions.NotFoundException): + _check_image_exists('image', docker_client) + + +def test_check_sh_entrypoint(docker_client): + assert _check_sh_entrypoint('image', docker_client) is None + docker_client.create_container.assert_called_with( + image='image', + command=['pip', 'show', 'scrapinghub-entrypoint-scrapy']) + docker_client.wait.return_value = 1 + with pytest.raises(shub_exceptions.NotFoundException): + _check_sh_entrypoint('image', docker_client) + + docker_client.wait.return_value = 0 + docker_client.logs.return_value = '' + with pytest.raises(shub_exceptions.NotFoundException): + _check_sh_entrypoint('image', docker_client) + + +def test_start_crawl(docker_client): + assert _check_start_crawl_entry('image', docker_client) is None + docker_client.create_container.assert_called_with( + image='image', command=['which', 'start-crawl']) + docker_client.wait.return_value = 1 + with pytest.raises(shub_exceptions.NotFoundException): + _check_start_crawl_entry('image', docker_client) + + docker_client.wait.return_value = 0 + docker_client.logs.return_value = '' + with pytest.raises(shub_exceptions.NotFoundException): + _check_start_crawl_entry('image', docker_client) + + +def test_run_docker_command(docker_client): + assert _run_docker_command( + docker_client, 'image-name', ['some', 'cmd']) == \ + (0, 'some-logs') + docker_client.create_container.assert_called_with( + image='image-name', command=['some', 'cmd']) + docker_client.start.assert_called_with({'Id': '12345'}) + docker_client.wait.assert_called_with(container='12345') + docker_client.logs.assert_called_with( + container='12345', stdout=True, stderr=False, + stream=False, timestamps=False) diff --git a/tests/test_upload.py b/tests/test_upload.py index a44d9ff..0ae7aed 100644 --- a/tests/test_upload.py +++ b/tests/test_upload.py @@ -1,11 +1,8 @@ -import os import mock from click.testing import CliRunner from unittest import TestCase from shub_image.upload import cli -from .utils import FakeProjectDirectory -from .utils import add_sh_fake_config class TestUploadCli(TestCase): diff --git a/tests/test_utils.py b/tests/test_utils.py index 18c3878..a9ecb93 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,17 +1,14 @@ import os import sys -import json import tempfile +from six import StringIO from unittest import TestCase import mock import click -import docker import pytest -import StringIO from shub import exceptions as shub_exceptions -from shub_image.utils import missing_modules from shub_image.utils import get_project_dir from shub_image.utils import get_docker_client from shub_image.utils import format_image_name @@ -29,13 +26,6 @@ class ReleaseUtilsTest(TestCase): - def test_missing_modules(self): - assert missing_modules() == [] - assert missing_modules('os', 'non-existing-module') == \ - ['non-existing-module'] - assert missing_modules('os', 'six', 'xxx11', 'xxx22') == \ - ['xxx11', 'xxx22'] - def test_get_project_dir(self): self.assertRaises( shub_exceptions.BadConfigException, get_project_dir) @@ -96,21 +86,6 @@ def test_format_image_name(self): mocked.return_value = config assert format_image_name('test', None) == 'test:test-version' - def test_custom_docker_client_workaround(self): - """Test workaround for https://github.com/docker/docker-py/issues/1059.""" - line = ( - '{"status":"Pulling from library/python","id":"2.7"}\r\n' - '{"status":"Pulling fs layer","progressDetail":{},"id":"5c90d4a2d1a8"}\r\n' - ) - - # mocked_docker.Client._stream_helper.return_value = (line,) - client = get_docker_client(validate=False) - with mock.patch.object(docker.Client, '_stream_helper', return_value=(line,)): - result = list(client._stream_helper(mock.Mock(), decode=False)) - assert len(result) == 2 - assert json.loads(result[0])['id'] == '2.7' - assert json.loads(result[1])['id'] == '5c90d4a2d1a8' - def test_get_credentials(self): assert get_credentials(insecure=True) == (None, None) assert get_credentials(apikey='apikey') == ('apikey', ' ') @@ -130,7 +105,7 @@ def test_init(self): def test_load(self): config = ReleaseConfig() - stream = StringIO.StringIO( + stream = StringIO( 'projects:\n dev: 123\n prod: 321\n' 'images:\n dev: registry/user/project\n prod: user/project\n' 'endpoints:\n dev: http://127.0.0.1/api/scrapyd/\n' diff --git a/tests/utils.py b/tests/utils.py index c723fee..10a2e2c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -37,6 +37,7 @@ def FakeProjectDirectory(): os.chdir(current) shutil.rmtree(tmpdir) + def add_scrapy_fake_config(tmpdir): # add fake scrapy.cfg config_path = os.path.join(tmpdir, 'scrapy.cfg') diff --git a/tox.ini b/tox.ini index db597a2..a51e16b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,16 @@ # tox.ini [tox] -envlist = py27 +envlist = py27,py33,py34,py35 [testenv] deps = -rrequirements.txt mock + flake8 pytest + pytest-cov Scrapy==1.0.5 install_command=pip install --process-dependency-links {opts} {packages} commands = - py.test + flake8 + py.test --cov=shub_image --cov-report= {posargs:shub_image tests}