CurrentEnvironmentBackend (#9)

CurrentEnvironmentBackend

arca/__init__.py

@@ -1,8 +1,9 @@
 from ._arca import Arca
-from .backend import BaseBackend, VenvBackend, DockerBackend
+from .backend import BaseBackend, VenvBackend, DockerBackend, CurrentEnvironmentBackend, RequirementsStrategy
 from .result import Result
 from .task import Task
 
 
-__all__ = ["Arca", "BaseBackend", "VenvBackend", "DockerBackend", "Result", "Task"]
+__all__ = ["Arca", "BaseBackend", "VenvBackend", "DockerBackend", "Result", "Task", "CurrentEnvironmentBackend",
+           "RequirementsStrategy"]
 __version__ = "0.0.1"

arca/backend/__init__.py

@@ -1,5 +1,7 @@
 from .base import BaseBackend
 from .venv import VenvBackend
 from .docker import DockerBackend
+from .current_environment import CurrentEnvironmentBackend, RequirementsStrategy
 
-__all__ = ["BaseBackend", "VenvBackend", "DockerBackend"]
+
+__all__ = ["BaseBackend", "VenvBackend", "DockerBackend", "CurrentEnvironmentBackend", "RequirementsStrategy"]

arca/backend/base.py

@@ -1,14 +1,20 @@
 import hashlib
+import json
+import os
 import re
+import stat
+import subprocess
+import sys
 from pathlib import Path
 from typing import Optional, Tuple
 
 from git import Repo
 
 import arca
+from arca.exceptions import BuildError
 from arca.result import Result
 from arca.task import Task
-from arca.utils import NOT_SET, LazySettingProperty
+from arca.utils import NOT_SET, LazySettingProperty, logger
 
 
 class BaseBackend:
@@ -21,6 +27,8 @@ def __init__(self, **settings):
         self._arca = None
         for key, val in settings.items():
             if hasattr(self, key) and isinstance(getattr(self, key), LazySettingProperty) and val is not NOT_SET:
+                if getattr(self, key).convert is not None:
+                    val = getattr(self, key).convert(val)
                 setattr(self, key, val)
 
     def inject_arca(self, arca):
@@ -63,3 +71,48 @@ def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Pat
 
     def get_or_create_environment(self, repo: str, branch: str, git_repo: Repo, repo_path: Path):  # pragma: no cover
         raise NotImplementedError
+
+
+class BaseRunInSubprocessBackend(BaseBackend):
+
+    def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Path) -> Result:
+        venv_path = self.get_or_create_environment(repo, branch, git_repo, repo_path)
+
+        script_name, script = self.create_script(task, venv_path)
+        script_path = Path(self._arca.base_dir, "scripts", script_name)
+        script_path.parent.mkdir(parents=True, exist_ok=True)
+
+        with script_path.open("w") as f:
+            f.write(script)
+
+        st = os.stat(str(script_path))
+        script_path.chmod(st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+        out_stream = b""
+        err_stream = b""
+
+        cwd = str(repo_path / self.cwd)
+
+        logger.info("Running at cwd %s", cwd)
+
+        try:
+            if venv_path is not None:
+                python_path = str(venv_path.resolve() / "bin" / "python")
+            else:
+                python_path = sys.executable
+
+            process = subprocess.Popen([python_path, str(script_path.resolve())],
+                                       stdout=subprocess.PIPE,
+                                       stderr=subprocess.PIPE,
+                                       cwd=cwd)
+
+            out_stream, err_stream = process.communicate()
+
+            return Result(json.loads(out_stream.decode("utf-8")))
+        except Exception as e:
+            logger.exception(e)
+            raise BuildError("The build failed", extra_info={
+                "exception": e,
+                "out_stream": out_stream,
+                "err_stream": err_stream,
+            })

arca/backend/current_environment.py

@@ -0,0 +1,125 @@
+import subprocess
+import sys
+from enum import Enum
+from pathlib import Path
+from typing import Optional, Iterable
+
+from git import Repo
+
+from arca.exceptions import ArcaMisconfigured, RequirementsMismatch, BuildError
+from arca.utils import LazySettingProperty, logger
+from .base import BaseRunInSubprocessBackend
+
+
+class RequirementsStrategy(Enum):
+    IGNORE = "ignore"
+    RAISE = "raise"
+    INSTALL_EXTRA = "install_extra"
+
+
+class CurrentEnvironmentBackend(BaseRunInSubprocessBackend):
+
+    current_environment_requirements = LazySettingProperty(key="current_environment_requirements")
+    requirements_strategy = LazySettingProperty(key="requirements_strategy",
+                                                default=RequirementsStrategy.RAISE,
+                                                convert=RequirementsStrategy)
+
+    def validate_settings(self):
+        super().validate_settings()
+
+        if self.current_environment_requirements is not None:
+            if not Path(self.current_environment_requirements).exists():
+                raise ArcaMisconfigured("Can't locate current environment requirements.")
+
+    def install_requirements(self, *, fl: Optional[Path]=None, requirements: Optional[Iterable[str]]=None,
+                             _action: str="install"):
+        if _action not in ["install", "uninstall"]:
+            raise ValueError(f"{_action} is invalid value for _invalid")
+
+        cmd = [sys.executable, "-m", "pip", _action]
+
+        if _action == "uninstall":
+            cmd += ["-y"]
+
+        if fl is not None:
+            cmd += ["-r", str(fl)]
+        elif requirements is not None:
+            cmd += list(requirements)
+        else:
+            raise ValueError("Either fl or requirements has to be provided")
+
+        logger.info("Installing requirements with command: %s", cmd)
+
+        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+        [out_stream, err_stream] = process.communicate()
+        out_stream = out_stream.decode("utf-8")
+        err_stream = err_stream.decode("utf-8")
+
+        logger.debug("Return code is %s", process.returncode)
+        logger.debug(out_stream)
+        logger.debug(err_stream)
+
+        if process.returncode:
+            raise BuildError(f"Unable to {_action} requirements from the target repository", extra_info={
+                "out_stream": out_stream,
+                "err_stream": err_stream,
+                "returncode": process.returncode
+            })
+
+    def get_requirements_set(self, fl):
+        return set([x.strip() for x in fl.read_text().split("\n") if x.strip()])
+
+    def get_or_create_environment(self, repo: str, branch: str, git_repo: Repo, repo_path: Path):
+        """ Handles requirements difference based on configured strategy
+        """
+        if self.requirements_strategy == RequirementsStrategy.IGNORE:
+            logger.info("Requirements strategy is IGNORE")
+            return
+
+        requirements = repo_path / self.requirements_location
+
+        # explicitly configured there are no requirements for the current environment
+        if self.current_environment_requirements is None:
+
+            if not requirements.exists():
+                return  # no diff, since no requirements both in current env and repository
+
+            requirements_set = self.get_requirements_set(requirements)
+
+            if len(requirements_set):
+                if self.requirements_strategy == RequirementsStrategy.RAISE:
+                    raise RequirementsMismatch(f"There are extra requirements in repository {repo}, branch {branch}.",
+                                               diff=requirements.read_text())
+
+                self.install_requirements(fl=requirements)
+
+        # requirements for current environment configured and exists
+        else:
+            current_requirements = Path(self.current_environment_requirements)
+
+            current_requirements_set = self.get_requirements_set(current_requirements)
+
+            if not requirements.exists():
+                return  # no req. file in repo -> no extra requirements
+
+            requirements_set = self.get_requirements_set(requirements)
+
+            # only requirements that are extra in repository requirements matter
+            extra_requirements_set = requirements_set - current_requirements_set
+
+            if len(extra_requirements_set) == 0:
+                return  # no extra requirements in repository
+            else:
+                if self.requirements_strategy == RequirementsStrategy.RAISE:
+                    raise RequirementsMismatch(f"There are extra requirements in repository {repo}, branch {branch}.",
+                                               diff="\n".join(extra_requirements_set))
+
+                elif self.requirements_strategy == RequirementsStrategy.INSTALL_EXTRA:
+                    self.install_requirements(requirements=extra_requirements_set)
+
+    def _uninstall(self, *args):
+        """ For usage in tests to uninstall packages from the current environment
+        :param args: packages to uninstall
+        """
+        self.install_requirements(requirements=args, _action="uninstall")

arca/backend/venv.py

@@ -1,21 +1,16 @@
-import json
-import os
-import stat
+import subprocess
 from pathlib import Path
 from venv import EnvBuilder
 
-import subprocess
 from git import Repo
 
 import arca
-from arca.task import Task
-from arca.result import Result
-from arca.utils import logger
 from arca.exceptions import BuildError
-from .base import BaseBackend
+from arca.utils import logger
+from .base import BaseRunInSubprocessBackend
 
 
-class VenvBackend(BaseBackend):
+class VenvBackend(BaseRunInSubprocessBackend):
 
     def create_or_get_venv(self, path: Path):
         requirements_file = self.get_requirements_file(path)
@@ -68,40 +63,3 @@ def create_or_get_venv(self, path: Path):
 
     def get_or_create_environment(self, repo: str, branch: str, git_repo: Repo, repo_path: Path) -> Path:
         return self.create_or_get_venv(repo_path)
-
-    def run(self, repo: str, branch: str, task: Task, git_repo: Repo, repo_path: Path) -> Result:
-        venv_path = self.get_or_create_environment(repo, branch, git_repo, repo_path)
-
-        script_name, script = self.create_script(task, venv_path)
-        script_path = Path(self._arca.base_dir, "scripts", script_name)
-        script_path.parent.mkdir(parents=True, exist_ok=True)
-
-        with script_path.open("w") as f:
-            f.write(script)
-
-        st = os.stat(str(script_path))
-        script_path.chmod(st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
-
-        out_stream = b""
-        err_stream = b""
-
-        cwd = str(repo_path / self.cwd)
-
-        logger.info("Running at cwd %s", cwd)
-
-        try:
-            process = subprocess.Popen([str(venv_path.resolve() / "bin" / "python"), str(script_path.resolve())],
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE,
-                                       cwd=cwd)
-
-            out_stream, err_stream = process.communicate()
-
-            return Result(json.loads(out_stream.decode("utf-8")))
-        except Exception as e:
-            logger.exception(e)
-            raise BuildError("The build failed", extra_info={
-                "exception": e,
-                "out_stream": out_stream,
-                "err_stream": err_stream,
-            })

arca/exceptions.py

@@ -28,3 +28,10 @@ def __init__(self, *args, full_output=None,  **kwargs):
 
 class FileOutOfRangeError(ValueError, ArcaException):
     pass
+
+
+class RequirementsMismatch(ValueError, ArcaException):
+
+    def __init__(self, *args, diff=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.diff = diff

arca/utils.py

@@ -1,6 +1,6 @@
 import importlib
 import logging
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Callable
 
 from .exceptions import ArcaMisconfigured
 
@@ -24,9 +24,10 @@ def load_class(location: str) -> type:
 
 
 class LazySettingProperty:
-    def __init__(self, *, key, default=NOT_SET) -> None:
+    def __init__(self, *, key, default=NOT_SET, convert: Callable=None) -> None:
         self.key = key
         self.default = default
+        self.convert = convert
 
     def __set_name__(self, cls, name):
         self.name = name
@@ -35,6 +36,10 @@ def __get__(self, instance, cls):
         if instance is None or (hasattr(instance, "_arca") and instance._arca is None):
             return self
         result = instance.get_setting(self.key, self.default)
+
+        if self.convert is not None:
+            result = self.convert(result)
+
         setattr(instance, self.name, result)
         return result
 

tests/common.py

@@ -0,0 +1,57 @@
+import os
+
+
+if os.environ.get("TRAVIS", False):
+    BASE_DIR = "/home/travis/build/{}/test_loc".format(os.environ.get("TRAVIS_REPO_SLUG", "mikicz/arca"))
+else:
+    BASE_DIR = "/tmp/arca/test"
+
+RETURN_STR_FUNCTION = """
+def return_str_function():
+    return "Some string"
+"""
+
+SECOND_RETURN_STR_FUNCTION = """
+def return_str_function():
+    return "Some other string"
+"""
+
+RETURN_DJANGO_VERSION_FUNCTION = """
+import django
+
+def return_str_function():
+    return django.__version__
+"""
+
+RETURN_PYTHON_VERSION_FUNCTION = """
+import sys
+
+def return_python_version():
+    return "{}.{}.{}".format(sys.version_info.major, sys.version_info.minor, sys.version_info.micro)
+"""
+
+RETURN_IS_XSLTPROC_INSTALLED = """
+import subprocess
+
+def return_is_xsltproc_installed():
+    try:
+        return subprocess.Popen(["xsltpoc", "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).wait()
+    except:
+        return False
+"""
+
+RETURN_IS_LXML_INSTALLED = """
+def return_is_lxml_installed():
+    try:
+        import lxml
+        return True
+    except:
+        return False
+"""
+
+RETURN_PLATFORM = """
+import platform
+
+def return_platform():
+    return platform.dist()[0]
+"""