Merge branch 'master' into documentation

.flake8

@@ -0,0 +1,6 @@
+[flake8]
+max-line-length = 120
+ignore =
+    # remove when https://gitlab.com/pycqa/flake8/issues/118 is resolved
+    F401,
+    E501,

.travis.yml

@@ -14,8 +14,9 @@ sudo: required
 services:
 - docker
 install:
-- python setup.py install
 - pip install codecov
+- pip install -r requirements.txt  # so travis pip cache can actually cache something
+- python setup.py install
 before_script:
 - docker login -u "$DOCKER_HUB_USERNAME" -p "$DOCKER_HUB_PASSWORD"
 script:

arca/_arca.py

@@ -13,7 +13,7 @@
 from .backend import BaseBackend
 from .result import Result
 from .task import Task
-from .utils import load_class, Settings, NOT_SET, logger, LazySettingProperty
+from .utils import load_class, Settings, NOT_SET, logger, LazySettingProperty, is_dirty
 
 BackendDefinitionType = Union[type, BaseBackend, str]
 DepthDefinitionType = Optional[int]
@@ -309,6 +309,19 @@ def get_files(self, repo: str, branch: str, *,
 
         return git_repo, repo_path
 
+    def get_repo(self, repo: str, branch: str, *,
+                 depth: Optional[int] = None,
+                 shallow_since: Optional[date] = None,
+                 reference: Optional[Path] = None
+                 ) -> Repo:
+        """ Returns a :class:`Repo <git.repo.base.Repo>` instance for the branch.
+
+        See :meth:`run` for arguments descriptions.
+        """
+        git_repo, _ = self.get_files(repo, branch, depth=depth, shallow_since=shallow_since, reference=reference)
+
+        return git_repo
+
     def cache_key(self, repo: str, branch: str, task: Task, git_repo: Repo) -> str:
         """ The key used for storing results in cache.
         """
@@ -320,10 +333,10 @@ def cache_key(self, repo: str, branch: str, task: Task, git_repo: Repo) -> str:
     def is_dirty(self) -> bool:
         """
         Returns if the repository the code is launched from was modified in any way.
-        Returns False if not in a repository.
+        Returns False if not in a repository at all.
         """
         try:
-            return Repo(".", search_parent_directories=True).is_dirty(untracked_files=True)
+            return is_dirty(Repo(".", search_parent_directories=True))
         except InvalidGitRepositoryError:
             return False
 
@@ -361,10 +374,15 @@ def run(self, repo: str, branch: str, task: Task, *,
                                              reference=reference)
 
         def create_value():
+            logger.debug("Value not in cache, creating.")
             return self.backend.run(repo, branch, task, git_repo, repo_path)
 
+        cache_key = self.cache_key(repo, branch, task, git_repo)
+
+        logger.debug("Cache key is %s", cache_key)
+
         return self.region.get_or_create(
-            self.cache_key(repo, branch, task, git_repo),
+            cache_key,
             create_value,
             should_cache_fn=self.should_cache_fn
         )

arca/backend/base.py

@@ -139,9 +139,9 @@ def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Pat
         script_name, script = self.create_script(task)
         script_path = Path(self._arca.base_dir, "scripts", script_name)
         script_path.parent.mkdir(parents=True, exist_ok=True)
+        script_path.write_text(script)
 
-        with script_path.open("w") as f:
-            f.write(script)
+        logger.info("Stored task script at %s", script_path)
 
         st = os.stat(str(script_path))
         script_path.chmod(st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
@@ -154,13 +154,18 @@ def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Pat
         logger.info("Running at cwd %s", cwd)
 
         try:
+            logger.debug("Running with python %s", python_path)
+
             process = subprocess.Popen([python_path, str(script_path.resolve())],
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE,
                                        cwd=cwd)
 
             out_stream, err_stream = process.communicate()
 
+            logger.debug("stdout output from the command")
+            logger.debug(out_stream)
+
             return Result(json.loads(out_stream.decode("utf-8")))
         except Exception as e:
             logger.exception(e)

arca/backend/docker.py

@@ -456,6 +456,9 @@ def push_to_registry(self, image: Image, image_tag: str):
 
         :raise PushToRegistryError: If the push fails.
         """
+        # already tagged, so it's already pushed
+        if f"{self.push_to_registry_name}:{image_tag}" in image.tags:
+            return
         image.tag(self.push_to_registry_name, image_tag)
 
         result = self.client.images.push(self.push_to_registry_name, image_tag)
@@ -471,6 +474,7 @@ def push_to_registry(self, image: Image, image_tag: str):
         last_line = json.loads(result.split("\n")[-1])
 
         if "error" in last_line:
+            self.client.images.remove(f"{self.push_to_registry_name}:{image_tag}")
             raise PushToRegistryError(f"Push of the image failed because of: {last_line['error']}", full_output=result)
 
         logger.info("Pushed image to registry %s:%s", self.push_to_registry_name, image_tag)
@@ -586,7 +590,13 @@ def get_image_for_repo(self, repo: str, branch: str, git_repo: Repo, repo_path:
         image_tag = self.get_image_tag(requirements_file, dependencies)
 
         if self.image_exists(image_name, image_tag):
-            return self.get_image(image_name, image_tag)
+            image = self.get_image(image_name, image_tag)
+
+            # in case the push to registry was set later and the image wasn't pushed when built
+            if self.push_to_registry_name is not None:
+                self.push_to_registry(image, image_tag)
+
+            return image
 
         if self.push_to_registry_name is not None:
             # the target image might have been built and pushed in a previous run already, let's try to pull it

arca/exceptions.py

@@ -44,7 +44,7 @@ class PushToRegistryError(ArcaException):
     """ Raised if pushing of images to Docker registry in :class:`DockerBackend` fails.
     """
 
-    def __init__(self, *args, full_output=None,  **kwargs):
+    def __init__(self, *args, full_output=None, **kwargs):
         super().__init__(*args, **kwargs)
 
         #: Full output of the push command

arca/utils.py

@@ -2,6 +2,8 @@
 import logging
 from typing import Any, Dict, Optional, Callable
 
+from git import Repo
+
 from .exceptions import ArcaMisconfigured
 
 
@@ -128,3 +130,19 @@ def __setitem__(self, key, value):
 
     def __contains__(self, item):
         return item in self.data
+
+
+def is_dirty(repo: Repo) -> bool:
+    """ Returns if the ``repo`` has been modified (including untracked files).
+    """
+    return repo.is_dirty(untracked_files=True)
+
+
+def get_last_commit_modifying_files(repo: Repo, *files) -> str:
+    """ Returns the hash of the last commit which modified some of the files.
+
+    :param repo: The repo to check in.
+    :param files: List of files to check
+    :return: Commit hash.
+    """
+    return repo.git.log(*files, n=1, format="%H")

pytest.ini

@@ -1,4 +1,3 @@
 [pytest]
 # -s is required to test vagrant - fabric needs stdin not to be captured
 addopts = -s --flake8 --cov=./
-flake8-max-line-length = 120

requirements.txt

@@ -9,6 +9,7 @@ requests
 python-vagrant
 fabric3
 entrypoints>=0.2.3
+flake8
 sphinx
 sphinx-autobuild
 sphinx_rtd_theme

tests/common.py

@@ -7,11 +7,6 @@
     BASE_DIR = "/tmp/arca/test"
 
 
-def replace_text(path, text):
-    with path.open("w") as fl:
-        fl.write(text)
-
-
 RETURN_STR_FUNCTION = """
 def return_str_function():
     return "Some string"
@@ -34,11 +29,11 @@ def return_str_function(*, kwarg):
     return kwarg[::-1]
 """
 
-RETURN_DJANGO_VERSION_FUNCTION = """
-import django
+RETURN_COLORAMA_VERSION_FUNCTION = """
+import colorama
 
 def return_str_function():
-    return django.__version__
+    return colorama.__version__
 """
 
 RETURN_PYTHON_VERSION_FUNCTION = """
@@ -48,23 +43,11 @@ def return_python_version():
     return "{}.{}.{}".format(sys.version_info.major, sys.version_info.minor, sys.version_info.micro)
 """
 
-RETURN_IS_XSLTPROC_INSTALLED = """
-import subprocess
-
-def return_is_xsltproc_installed():
-    try:
-        return subprocess.Popen(["xsltpoc", "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).wait()
-    except:
-        return False
-"""
+RETURN_FREETYPE_VERSION = """
+import freetype
 
-RETURN_IS_LXML_INSTALLED = """
-def return_is_lxml_installed():
-    try:
-        import lxml
-        return True
-    except:
-        return False
+def return_freetype_version():
+    return freetype.version()
 """
 
 RETURN_PLATFORM = """

tests/conftest.py

@@ -1,6 +1,7 @@
 import shutil
 import tempfile
 from pathlib import Path
+from uuid import uuid4
 
 import pytest
 from collections import namedtuple

tests/test_arca_class.py

@@ -1,16 +1,15 @@
 # encoding=utf-8
+import shutil
+from datetime import datetime, timedelta, date
 from pathlib import Path
 from uuid import uuid4
 
 import pytest
-import shutil
-
-from datetime import datetime, timedelta, date
 from git import Repo
 
-from arca import Arca, VenvBackend, CurrentEnvironmentBackend
+from arca import Arca, VenvBackend
 from arca.exceptions import ArcaMisconfigured, FileOutOfRangeError, PullError
-from common import BASE_DIR, replace_text
+from common import BASE_DIR
 
 
 def test_arca_backend():
@@ -122,7 +121,7 @@ def test_depth(temp_repo_static):
     arca = Arca(base_dir=BASE_DIR)
 
     for _ in range(19):  # since one commit is made in the fixture
-        replace_text(temp_repo_static.fl, str(uuid4()))
+        temp_repo_static.fl.write_text(str(uuid4()))
         temp_repo_static.repo.index.add([str(temp_repo_static.fl)])
         temp_repo_static.repo.index.commit("Initial")
 
@@ -133,7 +132,7 @@ def test_depth(temp_repo_static):
 
     # test when pulled again, the depth is increased since the local copy is stored
 
-    replace_text(temp_repo_static.fl, str(uuid4()))
+    temp_repo_static.fl.write_text(str(uuid4()))
     temp_repo_static.repo.index.add([str(temp_repo_static.fl)])
     temp_repo_static.repo.index.commit("Initial")
 
@@ -150,7 +149,7 @@ def test_depth(temp_repo_static):
 
     # test when pulled again, the depth setting is ignored
 
-    replace_text(temp_repo_static.fl, str(uuid4()))
+    temp_repo_static.fl.write_text(str(uuid4()))
     temp_repo_static.repo.index.add([str(temp_repo_static.fl)])
     temp_repo_static.repo.index.commit("Initial")
 
@@ -276,7 +275,7 @@ def test_shallow_since(temp_repo_static):
     now = datetime.now()
 
     for i in range(19, 0, -1):
-        replace_text(temp_repo_static.fl, str(uuid4()))
+        temp_repo_static.fl.write_text(str(uuid4()))
         temp_repo_static.repo.index.add([str(temp_repo_static.fl)])
         temp_repo_static.repo.index.commit(
             "Initial",
@@ -311,28 +310,6 @@ def test_shallow_since_validate(temp_repo_static, shallow_since, valid):
                                  shallow_since=shallow_since)
 
 
-def test_is_dirty():
-    arca = Arca(backend=CurrentEnvironmentBackend(
-        verbosity=2,
-        current_environment_requirements=None,
-        requirements_strategy="ignore"
-    ), base_dir=BASE_DIR)
-
-    if arca.is_dirty():
-        pytest.skip("Can't test is_dirty method when the current repo is dirty.")
-
-    assert not arca.is_dirty()
-
-    fl = Path(str(uuid4()))
-    fl.touch()
-
-    assert arca.is_dirty()
-
-    fl.unlink()
-
-    assert not arca.is_dirty()
-
-
 def test_pull_error():
     arca = Arca(base_dir=BASE_DIR)
 
@@ -357,3 +334,11 @@ def test_pull_error():
 
     with pytest.raises(PullError):
         arca.get_files(git_url, "master")
+
+
+def test_get_repo(temp_repo_static):
+    arca = Arca(base_dir=BASE_DIR)
+
+    pulled_repo = arca.get_repo(temp_repo_static.url, temp_repo_static.branch)
+
+    assert pulled_repo.head.object.hexsha == temp_repo_static.repo.head.object.hexsha