Attempt to merge Pulsar Docker images (1 / ...)

docker/coexecutor/Dockerfile

@@ -8,17 +8,31 @@ ENV PULSAR_CONFIG_CONDA_PREFIX /usr/local
 # bzip2 for Miniconda.
 # TODO: pycurl stuff...
 RUN apt-get update \
+    # Install CVMFS client
     && apt-get install -y --no-install-recommends lsb-release wget \
-        gcc python-setuptools \
+    && wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest_all.deb \
+    && dpkg -i cvmfs-release-latest_all.deb \
+    && rm -f cvmfs-release-latest_all.deb \
+    # Install packages
+    && apt-get update \
+    && apt-get install -y --no-install-recommends gcc python-setuptools \
         python-dev python-pip \
+        cvmfs cvmfs-config-default \
+        slurm-llnl slurm-drmaa-dev \
         bzip2 \
+    # Install Pulsar Python requirements
+    && pip install --no-cache-dir -U pip \
+    && pip install --no-cache-dir drmaa wheel kombu pykube poster \
+            webob psutil PasteDeploy six pyyaml paramiko \
+    # Remove build deps and cleanup
+    && apt-get -y remove python-dev gcc wget lsb-release \
     && apt-get -y autoremove \
     && apt-get autoclean \
-    && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log
-
-RUN pip install -U pip && pip install wheel kombu pykube poster
+    && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \
+    && /usr/sbin/create-munge-key
 
 ADD pulsar_app-*.dev0-py2.py3-none-any.whl /pulsar_app-*.dev0-py2.py3-none-any.whl
 
-RUN pip install /pulsar_app-*.dev0-py2.py3-none-any.whl[galaxy_extended_metadata]
+RUN pip install --no-cache-dir /pulsar_app-*.dev0-py2.py3-none-any.whl[galaxy_extended_metadata]
+RUN _pulsar-configure-galaxy-cvmfs
 RUN _pulsar-conda-init --conda_prefix=/pulsar_dependencies/conda

docker/cvmfs/Dockerfile

@@ -41,12 +41,9 @@ RUN python setup.py install \
     && pulsar-config --auto_conda --host 0.0.0.0 \
     && echo "export DRMAA_LIBRARY_PATH=/usr/lib/slurm-drmaa/lib/libdrmaa.so" >> local_env.sh \
     && cp docker/cvmfs/app.yml . \
-    && cp ./docker/testing/configure_slurm.py /usr/sbin/configure_slurm.py \
     && chown -R pulsar:pulsar /pulsar \
     && chmod +x /usr/local/bin/pulsar \
-    && cp /pulsar/docker/cvmfs/default.local /etc/cvmfs/ \
-    && cp /pulsar/docker/cvmfs/galaxyproject.org.conf /etc/cvmfs/domain.d/ \
-    && cp /pulsar/docker/cvmfs/data.galaxyproject.org.pub /etc/cvmfs/keys/ \
+    && _pulsar-configure-galaxy-cvmfs \
     && chown pulsar -R /usr/local
 
 # Pulsar will listen on this port
@@ -55,5 +52,5 @@ EXPOSE 8913
 # Must run CVMFS setup otherwise autofs does not get configured nor automount
 # starts. Then start Slurm and switch to a less-priviledged user for Pulsar.
 CMD /usr/bin/cvmfs_config setup; service munge start; \
-    python /usr/sbin/configure_slurm.py; service slurmd start; \
+    _pulsar-configure-slurm; service slurmd start; \
     service slurmctld start; su pulsar -c "/usr/local/bin/pulsar"

install_test/common_functions.bash

@@ -64,7 +64,13 @@ check_pulsar() {
     echo "Starting Pulsar in daemon mode."
     pulsar --daemon
     echo "Waiting for Pulsar to start."
+    while ! curl -s "http://localhost:$PULSAR_TARGET_POORT";
+    do
+        printf "."
+        sleep 1;
+    done
     sleep 2
+    cat paster.log
 
     cd ..    
 }

install_test/test_install.bash

@@ -11,7 +11,7 @@ init_pulsar
 
 cd pulsar
 echo "Running pulsar-config with default arguments"
-pulsar-config
+pulsar-config --auto_conda
 cd ..
 
 check_pulsar

pulsar/core.py

@@ -11,11 +11,11 @@
 from galaxy.objectstore import build_object_store_from_config
 try:
     # If galaxy-tool-util or Galaxy >=19.09 present.
-    from galaxy.tools.deps import build_dependency_manager
+    from galaxy.tool_util.deps import build_dependency_manager
     DependencyManager = None
 except ImportError:
     # If galaxy-lib or Galaxy <19.05 present.
-    from galaxy.tool_util.deps import DependencyManager
+    from galaxy.tools.deps import DependencyManager
     build_dependency_manager = None
 try:
     # If galaxy-lib or Galaxy <19.05 present.
@@ -142,10 +142,11 @@ def __setup_object_store(self, conf):
         self.object_store = build_object_store_from_config(object_store_config, config_dict=config_dict)
 
     def __setup_dependency_manager(self, conf):
+        default_tool_dependency_dir = "dependencies"
         if build_dependency_manager is not None:
-            self.dependency_manager = build_dependency_manager(app_config_dict=conf)
+            self.dependency_manager = build_dependency_manager(app_config_dict=conf, default_tool_dependency_dir=default_tool_dependency_dir)
         else:
-            dependencies_dir = conf.get("tool_dependency_dir", "dependencies")
+            dependencies_dir = conf.get("tool_dependency_dir", default_tool_dependency_dir)
             resolvers_config_file = conf.get("dependency_resolvers_config_file", "dependency_resolvers_conf.xml")
             conda_config = {k: v for k, v in conf.items() if k.startswith("conda_")}
             self.dependency_manager = DependencyManager(dependencies_dir, resolvers_config_file, app_config=conda_config)

pulsar/scripts/_configure_galaxy_cvmfs.py

@@ -0,0 +1,13 @@
+import pkg_resources
+import shutil
+
+
+def main():
+    config_path = pkg_resources.resource_filename('pulsar.scripts', 'cvmfs_data/default.local')
+    key_path = pkg_resources.resource_filename('pulsar.scripts', 'cvmfs_data/data.galaxyproject.org.pub')
+    domain_path = pkg_resources.resource_filename('pulsar.scripts', 'cvmfs_data/galaxyproject.org.conf')
+
+    # copy2?
+    shutil.copy(config_path, "/etc/cvmfs/")
+    shutil.copy(key_path, "/etc/cvmfs/keys/")
+    shutil.copy(domain_path, "/etc/cvmfs/domain.d/")

docker/testing/configure_slurm.py → pulsar/scripts/_configure_slurm.py

@@ -6,7 +6,6 @@
 from multiprocessing import cpu_count
 from socket import gethostname
 from string import Template
-from subprocess import call
 
 SLURM_CONFIG_TEMPLATE = '''
 # slurm.conf file generated by configurator.html.
@@ -92,5 +91,6 @@ def main():
     config_contents = Template(SLURM_CONFIG_TEMPLATE).substitute(template_params)
     open("/etc/slurm-llnl/slurm.conf", "w").write(config_contents)
 
+
 if __name__ == "__main__":
     main()

requirements.txt

@@ -1,11 +1,11 @@
 webob
 psutil
 PasteDeploy
-six
+six==1.11.0
 pyyaml
 galaxy-job-metrics>=19.9.0.dev1
 galaxy-objectstore>=19.9.0.dev1
-galaxy-tool-util>=19.9.0.dev1
+galaxy-tool-util>=19.9.0.dev3
 paramiko
 
 # Problematic for Python 3.

setup.py

@@ -101,18 +101,21 @@
         pulsar-submit=pulsar.scripts.submit:main
         pulsar-run=pulsar.scripts.run:main
         _pulsar-conda-init=pulsar.scripts._conda_init:main
+        _pulsar-configure-slurm=pulsar.scripts._configure_slurm:main
+        _pulsar-configure-galaxy-cvmfs=pulsar.scripts._configure_galaxy_cvmfs:main
     ''',
     scripts=scripts,
     package_data={'pulsar': [
         'managers/util/job_script/DEFAULT_JOB_FILE_TEMPLATE.sh',
         'managers/util/job_script/CLUSTER_SLOTS_STATEMENT.sh',
+        'scripts/cvmfs_data/*',
     ]},
     package_dir={'pulsar': 'pulsar'},
     include_package_data=True,
     install_requires=requirements,
     extras_require={
         ':python_version=="2.7"': py27_requirements,
-        'galaxy_extended_metadata': ['galaxy-job-execution>=19.9.0.dev0'],
+        'galaxy_extended_metadata': ['galaxy-job-execution>=19.9.0.dev0', 'galaxy-util[template]'],
     },
     license="Apache License 2.0",
     zip_safe=False,