Merge 66a827e into a053e29

.coveralls.yml

@@ -0,0 +1,2 @@
+service_name: travis-pro
+repo_token: wp9nvBKUHGovzWMFCrL6dTaFu6ryq37EP

.travis.yml

@@ -0,0 +1,28 @@
+language: python
+dist: bionic
+# addons:
+#   apt:
+#     sources:
+#       -	docker-bionic
+#     packages:
+#       - docker
+services:
+  - docker
+  - ssh
+python:
+  - 3.7
+  - 3.8
+branches:
+  only:
+    - master
+install:
+  - pip install -e .
+  - pip install pyopenssl[security]
+  - pip install -r canine/test/requirements.txt
+  - pip install coveralls
+
+script:
+  - sudo -E bash -c ". /home/travis/virtualenv/python3.7.6/bin/activate && coverage run --source=canine --omit='canine/backends/dummy/controller.py,*canine/test/*.py,canine/localization/delocalization.py,canine/__main__.py,canine/xargs.py' -m unittest discover canine/test"
+
+after_success:
+  - coveralls

canine/__main__.py

@@ -206,7 +206,7 @@ def xargs():
     for i in range(cmd.count(args.replacement_string)):
         cmd = cmd.replace(
             args.replacement_string,
-            'canine_arg{}'.format(i),
+            '$canine_arg{}'.format(i),
             1
         )
     Xargs(cmd, inputs, backend, args.name, args.working_directory, resources).run_pipeline()

canine/backends/__init__.py

@@ -7,6 +7,7 @@
 from .base import AbstractTransport, AbstractSlurmBackend
 from .local import LocalTransport, LocalSlurmBackend
 from .remote import RemoteTransport, RemoteSlurmBackend
+from .dummy import DummyTransport, DummySlurmBackend
 from .gcpTransient import TransientGCPSlurmBackend
 from .imageTransient import TransientImageSlurmBackend
 from .dockerTransient import DockerTransientImageSlurmBackend
@@ -16,6 +17,8 @@
     'LocalTransport',
     'RemoteSlurmBackend',
     'RemoteTransport',
+    'DummySlurmBackend',
+    'DummyTransport',
     'TransientGCPSlurmBackend',
     'TransientImageSlurmBackend',
     'DockerTransientImageSlurmBackend'

canine/backends/base.py

@@ -91,7 +91,7 @@ def mkdir(self, path: str):
         pass
 
     @abc.abstractmethod
-    def stat(self, path: str) -> typing.Any:
+    def stat(self, path: str, follow_symlinks: bool = True) -> typing.Any:
         """
         Returns stat information
         """
@@ -494,5 +494,6 @@ def estimate_cost(self, clock_uptime: typing.Optional[float] = None, node_uptime
         overhead for uptime (ie: controller nodes).
         Note: Job cost estimates may not sum up to the total cluster cost if the
         cluster was not at full utilization.
+        All units are in hours
         """
         return 0, None

canine/backends/dockerTransient.py

@@ -26,8 +26,12 @@ def __init__(
         compute_script = "/usr/local/share/slurm_gcp_docker/src/provision_worker_container_host.sh",
         nfs_disk_size = 2000, nfs_disk_type = "pd-standard", nfs_action_on_stop = "stop", nfs_image = "",
         action_on_stop = "delete", image_family = "slurm-gcp-docker", image = None,
-        clust_frac = 0.01, user = os.environ["USER"], **kwargs
+        clust_frac = 0.01, user = os.environ.get("USER"), **kwargs
     ):
+        if user is None:
+            # IE: USER was not set
+            raise ValueError("USER not set in environment. Must explicitly pass user argument")
+
         if "image" not in kwargs:
             kwargs["image"] = image
 

canine/backends/dummy/Dockerfile

@@ -0,0 +1,34 @@
+FROM gcr.io/broad-getzlab-workflows/slurm_gcp_docker_base
+MAINTAINER Aaron Graubert
+
+RUN apt-get update && apt-get install -y \
+  apt-transport-https \
+  ca-certificates \
+  curl \
+  wget \
+  gnupg-agent \
+  vim \
+  libmysqlclient-dev \
+  ssh \
+  software-properties-common
+
+# Munge fixes
+RUN echo exit 0 > /usr/sbin/policy-rc.d
+
+# Docker
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && \
+  apt-key fingerprint 0EBFCD88 && echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu disco stable" > /etc/apt/sources.list.d/docker-ce.list && \
+  apt-get update && apt-get install -y docker-ce
+
+# Python packages
+RUN pip3 install psutil
+
+COPY controller.py /
+COPY worker.sh /
+COPY conf /conf_templates
+COPY docker /usr/local/bin
+
+RUN chmod 755 controller.py worker.sh /usr/local/bin/docker
+
+ENTRYPOINT
+CMD /bin/bash