Merge 9261122 into e3957f6

.travis.yml

@@ -13,19 +13,19 @@ matrix:
   include:
     - name: "Python 3.6 support"
       python: 3.6
-      env: SLURM_CENTOS_VERSION=7 SLURM_VERSION=17.02.10 PIP_INSTALL=0 REQUIREMENTS_DEV_TXT=1 REQUIREMENTS_TEST_TXT=0 CONDA_DEV_ENV=0 PYTEST_ONLY=0 RUN_TESTS=1 BUILD_COVERAGE=1 BUILD_DOCS=1 DEPLOY_DOCS=1 DEPLOY_PYPI=0 DEPLOY_COVERAGE=1
+      env: CENTOS_VERSION=7 SLURM_VERSION=17.02.10 REMOTE_PYTHON_VERSION=3.6 TEST_JOBS=6 PIP_INSTALL=0 REQUIREMENTS_DEV_TXT=1 REQUIREMENTS_TEST_TXT=0 CONDA_DEV_ENV=0 PYTEST_ONLY=0 RUN_TESTS=1 BUILD_COVERAGE=1 BUILD_DOCS=1 DEPLOY_DOCS=1 DEPLOY_PYPI=0 DEPLOY_COVERAGE=1
     - name: "Python 3.5 support"
       python: 3.5
-      env: SLURM_CENTOS_VERSION=7 SLURM_VERSION=17.02.10 PIP_INSTALL=0 REQUIREMENTS_DEV_TXT=1 REQUIREMENTS_TEST_TXT=0 CONDA_DEV_ENV=0 PYTEST_ONLY=0 RUN_TESTS=1 BUILD_COVERAGE=1 BUILD_DOCS=1 DEPLOY_DOCS=0 DEPLOY_PYPI=0 DEPLOY_COVERAGE=0
+      env: CENTOS_VERSION=7 SLURM_VERSION=17.02.10 REMOTE_PYTHON_VERSION=3.5 TEST_JOBS=6 PIP_INSTALL=0 REQUIREMENTS_DEV_TXT=1 REQUIREMENTS_TEST_TXT=0 CONDA_DEV_ENV=0 PYTEST_ONLY=0 RUN_TESTS=1 BUILD_COVERAGE=1 BUILD_DOCS=1 DEPLOY_DOCS=0 DEPLOY_PYPI=0 DEPLOY_COVERAGE=0
     - name: "Pip install support"
       python: 3.6.6
-      env: SLURM_CENTOS_VERSION=7 SLURM_VERSION=17.02.10 PIP_INSTALL=1 REQUIREMENTS_DEV_TXT=0 REQUIREMENTS_TEST_TXT=1 CONDA_DEV_ENV=0 PYTEST_ONLY=1 RUN_TESTS=0 BUILD_COVERAGE=0 BUILD_DOCS=0 DEPLOY_DOCS=0 DEPLOY_PYPI=1 DEPLOY_COVERAGE=0
+      env: CENTOS_VERSION=7 SLURM_VERSION=17.02.10 REMOTE_PYTHON_VERSION=3.6 TEST_JOBS=6 PIP_INSTALL=1 REQUIREMENTS_DEV_TXT=0 REQUIREMENTS_TEST_TXT=1 CONDA_DEV_ENV=0 PYTEST_ONLY=1 RUN_TESTS=0 BUILD_COVERAGE=0 BUILD_DOCS=0 DEPLOY_DOCS=0 DEPLOY_PYPI=1 DEPLOY_COVERAGE=0
     - name: "CentOS 6 support"
       python: 3.6
-      env: SLURM_CENTOS_VERSION=6 SLURM_VERSION=17.02.10 PIP_INSTALL=0 REQUIREMENTS_DEV_TXT=1 REQUIREMENTS_TEST_TXT=0 CONDA_DEV_ENV=0 PYTEST_ONLY=1 RUN_TESTS=0 BUILD_COVERAGE=0 BUILD_DOCS=0 DEPLOY_DOCS=0 DEPLOY_PYPI=0 DEPLOY_COVERAGE=0
+      env: CENTOS_VERSION=6 SLURM_VERSION=17.02.10 REMOTE_PYTHON_VERSION=3.6 TEST_JOBS=6 PIP_INSTALL=0 REQUIREMENTS_DEV_TXT=1 REQUIREMENTS_TEST_TXT=0 CONDA_DEV_ENV=0 PYTEST_ONLY=1 RUN_TESTS=0 BUILD_COVERAGE=0 BUILD_DOCS=0 DEPLOY_DOCS=0 DEPLOY_PYPI=0 DEPLOY_COVERAGE=0
     - name: "Conda development support"
       python: 3.6
-      env: SLURM_CENTOS_VERSION=7 SLURM_VERSION=17.02.10 PIP_INSTALL=0 REQUIREMENTS_DEV_TXT=0 REQUIREMENTS_TEST_TXT=0 CONDA_DEV_ENV=1 PYTEST_ONLY=0 RUN_TESTS=1 BUILD_COVERAGE=1 BUILD_DOCS=1 DEPLOY_DOCS=0 DEPLOY_PYPI=0 DEPLOY_COVERAGE=0
+      env: CENTOS_VERSION=7 SLURM_VERSION=17.02.10 REMOTE_PYTHON_VERSION=3.6 TEST_JOBS=6 PIP_INSTALL=0 REQUIREMENTS_DEV_TXT=0 REQUIREMENTS_TEST_TXT=0 CONDA_DEV_ENV=1 PYTEST_ONLY=0 RUN_TESTS=1 BUILD_COVERAGE=1 BUILD_DOCS=1 DEPLOY_DOCS=0 DEPLOY_PYPI=0 DEPLOY_COVERAGE=0
 before_install:
   - source scripts/testing_setup/container_prepare_envs.sh
   - printenv | sort
@@ -57,10 +57,10 @@ install:
   - docker ps
 script:
   - if [ $PIP_INSTALL -eq 1 ]; then mv idact idact_tmp ; fi
-  - if [ $PYTEST_ONLY -eq 1 ]; then python -mpytest tests ; fi
+  - if [ $PYTEST_ONLY -eq 1 ]; then export IDACT_TESTING_PROCESS_COUNT=$TEST_JOBS && python -mpytest -n $TEST_JOBS tests ; fi
   - if [ $PIP_INSTALL -eq 1 ]; then mv idact_tmp idact ; fi
   - if [ $PIP_INSTALL -eq 1 ]; then idact-notebook --help ; fi
-  - if [ $RUN_TESTS -eq 1 ]; then unbuffer python scripts/run_tests.py ; fi
+  - if [ $RUN_TESTS -eq 1 ]; then unbuffer python scripts/run_tests.py -n $TEST_JOBS ; fi
   - if [ $BUILD_DOCS -eq 1 ]; then unbuffer python scripts/generate_diagrams.py ; fi
   - if [ $BUILD_DOCS -eq 1 ]; then unbuffer python scripts/build_docs.py --no-show ; fi
   - if [ $BUILD_COVERAGE -eq 1 ]; then unbuffer python scripts/view_coverage.py --no-show ; fi

CONTRIBUTING.md

@@ -0,0 +1,36 @@
+# Contributing
+
+## Guidelines [work in progress]
+
+### Submitting  issues
+
+Please include:
+ - description of the issue,
+ - steps to reproduce,
+ - the error message and/or log fragment,
+ - the local operating system and Python distribution,
+ - `conda env export` for Anaconda or `pip list` for `virtualenv` or similar,
+ - info about the cluster (if not Cyfronet/Prometheus),
+ - workaround if available.
+
+### Contributing changes
+
+Please:
+ - Fork `idact` on GitHub.
+ - Create a new branch off the `develop` branch.
+ - Run the testing setup in docker to test changes locally (see `scripts`).
+ - Add unit and/or functional tests.
+ - Submit a pull request to the develop branch.
+
+## Release instructions
+
+ - Bump version in `idact/__init__.py` and `setup.py` (`0.y-1 -> 0.y`).
+ - Update `docs/changelog.md`
+ - Update `docs/docs_by_version.md`
+ - Submit a pull request from develop branch to master.
+ - Commit title: `Version 0.y`, changelog in the message.
+ - Merge the pull request.
+ - Create a GitHub release on the branch master.
+ - Release (tag) name `0.y`, title: `Version 0.y`.
+ - Package will be released to pypi automatically.
+ - Upload zipped notebooks as assets if they changed since last release.

README.md

@@ -53,6 +53,13 @@ Python 3.5+.
 python -m pip install idact
 ```
 
+If you're using [Conda](https://conda.io/docs/), you may want to update
+your environment first:
+
+```
+conda update --all
+```
+
 ## Code samples
 
 ### Accessing a cluster

docs/changelog.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.7
+
+ - Documentation improvements.
+ - Use Docker container for testing setup.
+ - Improve `idact-notebook` help message.
+ - Remove unnecessary overheads in functional tests.
+ - Refactor functional tests to support parallel execution.
+
 ## 0.6
 
  - Added support for CentOS 6.

docs/contributing.rst

@@ -0,0 +1 @@
+.. mdinclude:: ../CONTRIBUTING.md

docs/docs_by_version.md

@@ -2,6 +2,7 @@
 
  - [master](https://garstka.github.io/idact/master/html/index.html)
  - [develop](https://garstka.github.io/idact/develop/html/index.html)
+ - [0.7](https://garstka.github.io/idact/0.7/html/index.html)
  - [0.6](https://garstka.github.io/idact/0.6/html/index.html)
  - [0.5.1](https://garstka.github.io/idact/0.5.1/html/index.html)
  - [0.5](https://garstka.github.io/idact/0.5/html/index.html)

docs/index.rst

@@ -14,6 +14,7 @@ For public API reference see :py:mod:`idact`.
    self
    notebooks
    api/idact
+   contributing
    development_overview
    prometheus
    diagrams

envs/environment-dev.yml

@@ -5,15 +5,15 @@ dependencies:
   - python=3.6
   - setuptools
   - pip
-  - "sphinx>=1.7.4"
-  - "pytest-cov>=2.5.1"
+  - "sphinx>=1.7.4,<=1.8.4"
   - "click>=6.7"
-  - "pytest>=3.5.1"
   - "python-dateutil>=2.7.2"
   - "graphviz>=2.38.0"
   - "nb_conda_kernels>=2.1.0"
   - pip:
     - "fabric3>=1.14"
+    - "pytest-cov>=2.5.1"
+    - "pytest>=4.4.0"
     - "pytest-flake8>=1.0.1"
     - "pytest-pylint>=0.9.0"
     - "m2r>=0.1.14"
@@ -33,3 +33,4 @@ dependencies:
     - "jupyter>=1.0.0"
     - "jupyterlab>=0.35.4"
     - "coveralls>=1.5.1"
+    - "pytest-xdist>=1.26.1"

idact/__init__.py

@@ -33,7 +33,7 @@
 
 __author__ = """Matt Garstka"""
 __email__ = 'matt@garstka.net'
-__version__ = '0.6'
+__version__ = '0.7'
 
 _IMPORTED = {add_cluster,
              load_environment,

idact/detail/auth/authenticate.py

@@ -116,7 +116,7 @@ def authenticate(host: str,
 
     previous_abort_on_prompts = env.abort_on_prompts
     env.shell = "/bin/bash --noprofile -l -c"
-    env.key = None
+    env.pop('key', None)  # Do not use an in-memory key.
     env.user = config.user
     env.abort_on_prompts = True
     log = get_logger(__name__)

idact/detail/auth/install_key.py

@@ -111,12 +111,12 @@ def task():
         with stage_debug(log, "Creating authorized keys file: %s",
                          authorized_keys_path):
             with capture_fabric_output_to_log():
-                run("mkdir -p ~/.ssh")
-                run("chmod 700 ~/.ssh")
-                run("touch '{authorized_keys_path}'".format(
-                    authorized_keys_path=authorized_keys_path))
-                run("chmod 644 '{authorized_keys_path}'".format(
-                    authorized_keys_path=authorized_keys_path))
+                run(
+                    "mkdir -p ~/.ssh"
+                    " && chmod 700 ~/.ssh"
+                    " && touch '{authorized_keys_path}'"
+                    " && chmod 644 '{authorized_keys_path}'".format(
+                        authorized_keys_path=authorized_keys_path))
 
         with stage_debug(log, "Downloading authorized keys file."):
             with capture_fabric_output_to_log():

idact/detail/auth/install_shared_home_key.py

@@ -32,8 +32,8 @@ def task():
             key if it's not been generated already."""
         with stage_info(log, "Creating the ssh directory."):
             with capture_fabric_output_to_log():
-                run("mkdir -p ~/.ssh")
-                run("chmod 700 ~/.ssh")
+                run("mkdir -p ~/.ssh"
+                    " && chmod 700 ~/.ssh")
 
         with capture_fabric_output_to_log():
             file_exists = exists(SHARED_HOST_KEY_PATH)

idact/detail/dask/create_scratch_dir.py

@@ -40,8 +40,10 @@ def create_scratch_subdir(node: NodeInternal) -> str:
         "{scratch}/{subdir}".format(
             scratch=scratch,
             subdir=SCRATCH_SUBDIR))
-    node.run("mkdir -p {}".format(scratch_subdir))
-    node.run("chmod 700 {}".format(scratch_subdir))
+    node.run(
+        "mkdir -p {scratch_subdir}"
+        " && chmod 700 {scratch_subdir}".format(
+            scratch_subdir=scratch_subdir))
     scratch_subdir_realpath = node.run("readlink -vf {}".format(
         scratch_subdir))
 

idact/detail/deployment/create_deployment_dir.py

@@ -20,8 +20,10 @@ def create_runtime_dir(node: NodeInternal) -> str:
     formatted_runtime_dir = DEPLOYMENT_RUNTIME_DIR_FORMAT.format(
         deployment_id=deployment_id)
 
-    node.run('mkdir -p {}'.format(formatted_runtime_dir))
-    node.run('chmod 700 {}'.format(formatted_runtime_dir))
+    node.run(
+        "mkdir -p {formatted_runtime_dir}"
+        " && chmod 700 {formatted_runtime_dir}".format(
+            formatted_runtime_dir=formatted_runtime_dir))
     runtime_dir = node.run("readlink -vf {}".format(formatted_runtime_dir))
 
     return runtime_dir

idact/detail/deployment/generic_deployment.py

@@ -9,8 +9,6 @@
 from idact.detail.serialization.serializable import Serializable
 from idact.detail.serialization.serializable_types import SerializableTypes
 
-CANCEL_TIMEOUT = 5
-
 
 class GenericDeployment(Serializable):
     """Deployment of a program on a node.
@@ -19,8 +17,6 @@ class GenericDeployment(Serializable):
 
         :param pid: Process id.
 
-        :param output: Initial script output.
-
         :param runtime_dir: Runtime dir to remove.
 
     """
@@ -55,20 +51,18 @@ def cancel(self):
 
         parent_pid = self._pid
         node = self._node
+        tree = ' '.join([str(pid)
+                         for pid
+                         in ptree(pid=parent_pid, node=node)])
 
         def cancel_task():
-            """Kills a list of processes, waits and fails if all are still
+            """Kills the process tree and fails if the parent is still
                 running after a timeout."""
-            tree = ' '.join([str(pid)
-                             for pid
-                             in ptree(pid=parent_pid, node=node)])
             node.run(
                 "kill {tree}"
-                "; sleep {timeout}"
-                "; kill -0 {pid} && exit 1 || exit 0".format(
+                "; kill -0 {parent_pid} && exit 1 || exit 0".format(
                     tree=tree,
-                    pid=self._pid,
-                    timeout=CANCEL_TIMEOUT))
+                    parent_pid=parent_pid))
 
         log = get_logger(__name__)
         with remove_runtime_dir_on_exit(node=self._node,

idact/detail/deployment_sync/deployment_definitions_serialization.py

@@ -82,8 +82,8 @@ def serialize_deployment_definitions_to_cluster(
         serialized = deployments.serialize()
         file_contents = json.dumps(serialized, sort_keys=True, indent=4)
         parent_path = get_deployment_definitions_parent_path(node=node)
-        node.run("mkdir -p {}".format(parent_path))
-        node.run("chmod 700 {}".format(parent_path))
+        node.run("mkdir -p {parent_path}"
+                 " && chmod 700 {parent_path}".format(parent_path=parent_path))
         path = get_deployment_definitions_file_path(node=node)
         put_file_on_node(node=node,
                          remote_path=path,

idact/detail/entry_point/fetch_port_info.py

@@ -48,7 +48,11 @@ def task():
             with capture_fabric_output_to_log():
                 with cd(dir_path):
                     files = run("echo *", pty=False)
-                    result.append(files)
+                    if files[0] != "*":
+                        result.append(files)
+                    else:
+                        log.warning("Port info files not found.")
+                        result.append("")
         else:
             log.warning("Port info directory not found.")
             result.append("")

idact/detail/entry_point/upload_entry_point.py

@@ -43,10 +43,10 @@ def task():
         """Creates the entry point dir and file.
             Fails if it couldn't be created."""
         with capture_fabric_output_to_log():
-            run("mkdir -p {entry_point_location}".format(
-                entry_point_location=entry_point_location))
-            run("chmod 700 {entry_point_location}".format(
-                entry_point_location=entry_point_location))
+            run(
+                "mkdir -p {entry_point_location}"
+                " && chmod 700 {entry_point_location}".format(
+                    entry_point_location=entry_point_location))
 
             file_name = get_random_file_name(
                 length=ENTRY_POINT_FILE_NAME_LENGTH)

idact/detail/helper/get_free_local_port.py

@@ -0,0 +1,12 @@
+import socket
+
+
+def get_free_local_port() -> int:
+    """Returns a free local port.
+
+        Binds a socket to port 0 and immediately releases it.
+
+    """
+    with socket.socket() as sock:
+        sock.bind(("", 0))
+        return sock.getsockname()[1]

idact/detail/helper/ports.py

@@ -0,0 +1,5 @@
+"""Unique reserved remote ports."""
+
+PORT_1 = 8081
+PORT_2 = 8082
+PORT_3 = 8083

idact/detail/helper/ptree.py

@@ -4,26 +4,32 @@
 
 from idact.core.nodes import Node
 
+# https://unix.stackexchange.com/questions/124127/kill-all-descendant-processes
+LIST_DESCENDANTS = (
+    r'list_descendants () {'
+    r' local children=$(pgrep -P "$1");'
+    r' for pid in $children;'
+    r' do list_descendants "$pid";'
+    r' done;'
+    r' echo "$children"; }')
+
 
 def ptree(pid: int, node: Node) -> List[int]:
-    """Returns a list containing this PID and all its offspring, by running
-        `pgrep` repeatedly.
+    """Returns a list containing this PID and all its descendants.
 
         :param pid: Parent process pid.
 
         :param node: Node to run pgrep on.
 
     """
-    result = node.run("pgrep -P {pid}; exit 0".format(pid=pid))
+    result = node.run(
+        "{list_descendants};"
+        " echo $(list_descendants {pid})".format(
+            list_descendants=LIST_DESCENDANTS,
+            pid=pid))
     if not result:
         return [pid]
-    child_pids = [int(child_pid)
-                  for child_pid in result.splitlines()]
-
-    rest = [ptree(child_pid, node)
-            for child_pid in child_pids]
-
-    rest_flat = [y for x in rest
-                 for y in x]
+    descendant_pids = [int(descendant_pid)
+                       for descendant_pid in result.split(' ')]
 
-    return [pid] + rest_flat
+    return [pid] + descendant_pids

idact/detail/jupyter_app/main.py

@@ -76,14 +76,18 @@
               nargs=2,
               multiple=True,
               type=str,
+              metavar='ARG VALUE',
               default=[],
-              help="Native arguments for the workload"
-                   " manager. Values are not validated. Supported arguments"
-                   " take precedence over native arguments. Arguments with"
-                   " None as value are treated as flags. Arguments specified"
-                   " later override earlier arguments. [Allocation parameter]."
-                   " Default: No native"
-                   " arguments.")
+              help="Native arguments for the workload manager."
+                   " Flags have value=None, e.g. --native-arg --flag None."
+                   " Can be repeated for multiple native args:"
+                   " --native-arg -arg1 v1 --native-arg -arg2 v2 (...)."
+                   " Values are not validated."
+                   " Supported arguments take precedence over native"
+                   " arguments."
+                   " Arguments specified later override earlier arguments."
+                   " [Allocation parameter]."
+                   " Default: No native arguments.")
 def main(cluster_name: str,
          environment: Optional[str],
          save_defaults: bool,