Merge pull request #16 from AndreasSko/fixes

Cleanup and fixes
bgruening · Mar 18, 2020 · 288432c · 288432c
2 parents 38f1dcd + 4eddab6
commit 288432c
Show file tree

Hide file tree

Showing 15 changed files with 69 additions and 1,851 deletions.
diff --git a/.github/workflows/compose-v2.yml b/.github/workflows/compose-v2.yml
@@ -6,16 +6,19 @@ jobs:
     strategy:
       matrix:
         image:
-          - galaxy-server
-          - galaxy-nginx
-          - galaxy-postgres
-          - galaxy-htcondor
-          - galaxy-slurm
-          - galaxy-slurm-node-discovery
-          - galaxy-configurator
-          - galaxy-bioblend-test
-          - galaxy-workflow-test
-          - galaxy-selenium-test
+          - name: galaxy-server
+          - name: galaxy-nginx
+          - name: galaxy-postgres
+          - name: galaxy-htcondor
+          - name: galaxy-slurm
+          - name: galaxy-slurm-node-discovery
+          - name: galaxy-configurator
+          - name: galaxy-bioblend-test
+            subdir: tests/
+          - name: galaxy-workflow-test
+            subdir: tests/
+          - name: galaxy-selenium-test
+            subdir: tests/
       fail-fast: false
     steps:
       - name: Checkout
@@ -39,11 +42,11 @@ jobs:
         run: |
           for i in 1 2; do
             docker buildx build  \
-            --cache-to "type=registry,mode=max,ref=docker.io/andreassko/${{ matrix.image }}:${{ steps.image_tag.outputs.image_tag }}_cache" \
-            --cache-from "type=registry,ref=docker.io/andreassko/${{ matrix.image }}:${{ steps.image_tag.outputs.image_tag }}_cache" \
-            --output "type=image,name=andreassko/${{ matrix.image }}:${{ steps.image_tag.outputs.image_tag }},push=true" \
+            --cache-to "type=registry,mode=max,ref=docker.io/andreassko/${{ matrix.image.name }}:${{ steps.image_tag.outputs.image_tag }}_cache" \
+            --cache-from "type=registry,ref=docker.io/andreassko/${{ matrix.image.name }}:${{ steps.image_tag.outputs.image_tag }}_cache" \
+            --output "type=image,name=andreassko/${{ matrix.image.name }}:${{ steps.image_tag.outputs.image_tag }},push=true" \
             --build-arg GALAXY_REPO=https://github.com/andreassko/galaxy \
-            ${{ matrix.image }} && break || echo "Fail.. Retrying"
+            ${{ matrix.image.subdir }}${{ matrix.image.name }} && break || echo "Fail.. Retrying"
           done;
         shell: bash
         working-directory: ./compose-v2
@@ -66,24 +69,24 @@ jobs:
             options: --scale slurm_node=3
         test:
           - name: bioblend
-            files: -f docker-compose.test.yml -f docker-compose.test.bioblend.yml
+            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.bioblend.yml
             exit-from: galaxy-bioblend-test
             second_run: "true"
           - name: workflow_ard
-            files: -f docker-compose.test.yml -f docker-compose.test.workflows.yml
+            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
             exit-from: galaxy-workflow-test
             workflow: sklearn/ard/ard.ga
             second_run: "true"
           - name: workflow_mapping_by_sequencing
-            files: -f docker-compose.test.yml -f docker-compose.test.workflows.yml
+            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
             exit-from: galaxy-workflow-test
             workflow: training/variant-analysis/mapping-by-sequencing/mapping_by_sequencing.ga
           - name: workflow_GC-lite
-            files: -f docker-compose.test.yml -f docker-compose.test.workflows.yml
+            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
             exit-from: galaxy-workflow-test
             workflow: GraphClust2/GC-lite.ga
           - name: selenium
-            files: -f docker-compose.test.yml -f docker-compose.test.selenium.yml
+            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.selenium.yml
             exit-from: galaxy-selenium-test
       fail-fast: false
     steps:
@@ -181,7 +184,7 @@ jobs:
             files: -f docker-compose.yml -f docker-compose.singularity.yml
         test:
           - name: workflow_example1
-            files: -f docker-compose.test.yml -f docker-compose.test.workflows.yml
+            files: -f tests/docker-compose.test.yml -f tests/docker-compose.test.workflows.yml
             exit-from: galaxy-workflow-test
             workflow: example1/wf3-shed-tools.ga
       fail-fast: false

diff --git a/compose-v2/docker-compose.yml b/compose-v2/docker-compose.yml
@@ -16,12 +16,14 @@ services:
       # This is the directory where all your files from Galaxy will be stored
       # on your host system
       - ${EXPORT_DIR:-./export}/:/export/:delegated
-      - /var/run/docker.sock:/var/run/docker.sock # TODO
+      - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
       - postgres
       - rabbitmq
     networks:
       - galaxy
+  # This is responsible for the whole configuration of your setup and should
+  # be the central place of configuration.
   galaxy-configurator:
     image: ${IMAGE_DOMAIN:-andreassko}/galaxy-configurator:${IMAGE_TAG:-latest}
     build: galaxy-configurator
@@ -32,18 +34,16 @@ services:
       - GALAXY_CONFIG_DATABASE_CONNECTION=postgresql://galaxy:chaopagoosaequuashie@postgres/galaxy
       - GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL=${GALAXY_CONFIG_GALAXY_INFRASTRUCTURE_URL:-http://localhost}
       - GALAXY_CONFIG_CONDA_AUTO_INSTALL=true
-      - GALAXY_AMQP_INTERNAL_CONNECTION=amqp://galaxy:vaiJa3ieghai2ief0jao@rabbitmq/galaxy
+      - GALAXY_CONFIG_AMQP_INTERNAL_CONNECTION=amqp://galaxy:vaiJa3ieghai2ief0jao@rabbitmq/galaxy
       - GALAXY_PROXY_PREFIX=${GALAXY_PROXY_PREFIX:-}
+      - GALAXY_CONFIG_CLEANUP_JOB=onsuccess
       - NGINX_OVERWRITE_CONFIG=true
     volumes:
       - ${EXPORT_DIR:-./export}/galaxy/config:/galaxy/config
       - ${EXPORT_DIR:-./export}/nginx:/etc/nginx
       - ./base_config.yml:/base_config.yml
       - ./galaxy-configurator/templates:/templates
   postgres:
-    # If you are using the official postgres image, it needs to be populated by calling
-    # docker-compose run galaxy install_db.sh
-    # on first run
     image: ${IMAGE_DOMAIN:-andreassko}/galaxy-postgres:${IMAGE_TAG:-latest}
     build: galaxy-postgres
     hostname: postgres

diff --git a/compose-v2/galaxy-server/Dockerfile b/compose-v2/galaxy-server/Dockerfile
@@ -84,12 +84,43 @@ RUN wget https://github.com/sylabs/singularity/releases/download/v${SINGULARITY_
     && make -C builddir \
     && /usr/bin/common_cleanup.sh
 
+# --- Final image ---
 FROM ubuntu:18.04 as final
+
 COPY ./files/common_cleanup.sh /usr/bin/common_cleanup.sh
+COPY ./files/create_galaxy_user.py /usr/local/bin/create_galaxy_user.py
+
+ENV EXPORT_DIR=/export \
+    GALAXY_ROOT=/galaxy \
+    HTCONDOR_ROOT=/opt/htcondor
+
+ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_19.05} \
+    GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \
+    GALAXY_STATIC_DIR=$GALAXY_ROOT/static \
+    GALAXY_EXPORT=$EXPORT_DIR/galaxy \
+    GALAXY_CONFIG_DIR=$GALAXY_ROOT/config \
+    GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=/tool_deps \
+    GALAXY_CONFIG_TOOL_PATH=$GALAXY_ROOT/tools \
+    GALAXY_VIRTUAL_ENV=$GALAXY_ROOT/.venv \
+    GALAXY_DATABASE_PATH=$GALAXY_ROOT/database
+
+ENV GALAXY_USER=galaxy \
+    GALAXY_GROUP=galaxy \
+    GALAXY_UID=1450 \
+    GALAXY_GID=1450 \
+    GALAXY_HOME=/home/galaxy
+
+ENV GALAXY_CONFIG_FILE=$GALAXY_CONFIG_DIR/galaxy.yml
+
+# Set permissions
+RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \
+    && useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" --shell /bin/bash $GALAXY_USER \
+    && /usr/bin/common_cleanup.sh
 
 # Install remaining dependencies
 RUN apt update && apt install --no-install-recommends ca-certificates gcc gnupg2 libgomp1 liblzma-dev libbz2-dev libpq-dev \
-                                                      mercurial make netcat python-dev python-setuptools python-pip zlib1g-dev -y \
+                                                      mercurial make netcat python-dev python-setuptools python-pip \
+                                                      squashfs-tools zlib1g-dev -y \
     && pip install Cython wheel \
     && pip install pysam \
     && /usr/bin/common_cleanup.sh
@@ -126,34 +157,12 @@ RUN apt update \
     && apt install --no-install-recommends cvmfs -y \
     && mkdir /srv/cvmfs \
     && /usr/bin/common_cleanup.sh
-
 COPY files/cvmfs /etc/cvmfs
 
-ENV EXPORT_DIR=/export \
-    GALAXY_ROOT=/galaxy \
-    HTCONDOR_ROOT=/opt/htcondor
-
-ENV GALAXY_RELEASE=${GALAXY_RELEASE:-release_19.05} \
-    GALAXY_REPO=${GALAXY_REPO:-https://github.com/galaxyproject/galaxy} \
-    GALAXY_STATIC_DIR=$GALAXY_ROOT/static \
-    GALAXY_EXPORT=$EXPORT_DIR/galaxy \
-    GALAXY_CONFIG_DIR=$GALAXY_ROOT/config \
-    GALAXY_CONFIG_TOOL_DEPENDENCY_DIR=/tool_deps \
-    GALAXY_CONFIG_TOOL_PATH=$GALAXY_ROOT/tools \
-    GALAXY_VIRTUAL_ENV=$GALAXY_ROOT/.venv \
-    GALAXY_DATABASE_PATH=$GALAXY_ROOT/database
-
-ENV GALAXY_USER=galaxy \
-    GALAXY_GROUP=galaxy \
-    GALAXY_UID=1450 \
-    GALAXY_GID=1450 \
-    GALAXY_HOME=/home/galaxy
-
-ENV GALAXY_CONFIG_FILE=$GALAXY_CONFIG_DIR/galaxy.yml
-
-# Set permissions
-RUN groupadd -r $GALAXY_USER -g $GALAXY_GID \
-    && useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -c "Galaxy user" --shell /bin/bash $GALAXY_USER \
+# Install Docker
+RUN apt update \
+    && apt install --no-install-recommends docker.io -y \
+    && usermod -aG docker $GALAXY_USER \
     && /usr/bin/common_cleanup.sh
 
 COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_galaxy ${GALAXY_ROOT} ${GALAXY_ROOT}
@@ -163,24 +172,13 @@ COPY --chown=$GALAXY_USER:$GALAXY_USER --from=build_miniconda /etc/profile.d/con
 
 # Install Singularity
 COPY --from=build_singularity /singularity /singularity
-RUN apt update && apt install squashfs-tools -y \
-    && make -C /singularity/builddir install \
+RUN make -C /singularity/builddir install \
     && rm -rf /singularity \
     && sed -e '/bind path = \/etc\/localtime/s/^/#/g' -i /usr/local/etc/singularity/singularity.conf \
     && /usr/bin/common_cleanup.sh
-# Install Docker
-RUN apt update \
-    && apt install --no-install-recommends docker.io -y \
-    && usermod -aG docker $GALAXY_USER \
-    && /usr/bin/common_cleanup.sh
 
-COPY ./files/job_conf.xml ${GALAXY_CONFIG_DIR}/job_conf.xml
-COPY ./files/galaxy.yml /${GALAXY_CONFIG_DIR}/galaxy.yml
 COPY ./files/start.sh /usr/bin/start.sh
 
-# Some helpful scripts
-COPY ./files/create_galaxy_user.py /usr/local/bin/create_galaxy_user.py
-
 EXPOSE 80
 
 ENTRYPOINT "/usr/bin/start.sh"