Merge pull request #25 from EGA-archive/images

EGA-archive Images
EGA-archive · Feb 18, 2019 · 08eb815 · 08eb815
2 parents 1f8f2f0 + 1351ce3
commit 08eb815
Show file tree

Hide file tree

Showing 17 changed files with 597 additions and 299 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -10,17 +10,37 @@ git:
   quiet: true
 
 stages:
+  - name: preflight
   - name: tests
+    if: type = pull_request
   - name: images
     if: type = pull_request AND branch = master
 
 jobs:
   include:
-    - stage: tests
+    #
+    # Unit Tests and Image Build
+    #
+    - stage: preflight
       name: "Unit Tests"
       python: 3.6
       before_script: pip install tox-travis
       script: tox
+    - stage: preflight
+      if: type = pull_request
+      services: docker
+      name: "Images build"
+      before_script:
+        - pip install tox-travis
+        - cd deploy/images
+        # Pulling image for layer caching
+        - make -j 2 pull-base pull-inbox IMGTAG=latest
+      script:
+        - make -j 4 base inbox unit-tests
+        - make -j 2 push-base push-inbox
+    #
+    # BATS Tests
+    #
     - stage: tests
       name: "Integration Tests"
       services: docker
@@ -29,6 +49,8 @@ jobs:
         - git clone https://github.com/bats-core/bats-core.git
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
+        - make -C images -j 2 pull-base pull-inbox
+        - make -C images -j 2 retag-base retag-inbox
         - make prepare
         - make bootstrap ARGS="--keyserver ega"
         - sudo chown -R travis private
@@ -45,6 +67,8 @@ jobs:
         - git clone https://github.com/bats-core/bats-core.git
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
+        - make -C images -j 2 pull-base pull-inbox
+        - make -C images -j 2 retag-base retag-inbox
         - make prepare
         - make bootstrap ARGS="--keyserver ega"
         - sudo chown -R travis private
@@ -65,6 +89,8 @@ jobs:
         - git clone https://github.com/bats-core/bats-core.git
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
+        - make -C images -j 2 pull-base pull-inbox
+        - make -C images -j 2 retag-base retag-inbox
         - make prepare
         - make bootstrap ARGS="--keyserver ega"
         - sudo chown -R travis private
@@ -73,11 +99,16 @@ jobs:
       script:
         - cd ../tests
         - bats security
+    #
+    # Cucumber Tests
+    #
     - stage: tests
       name: "Cucumber Ingestion Tests"
       before_script:
         - pip3 install -r requirements.txt
         - cd deploy
+        - make -C images -j 2 pull-base pull-inbox
+        - make -C images -j 2 retag-base retag-inbox
         - make prepare
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
@@ -91,6 +122,8 @@ jobs:
       before_script:
         - pip3 install -r requirements.txt
         - cd deploy
+        - make -C images -j 2 pull-base pull-inbox
+        - make -C images -j 2 retag-base retag-inbox
         - make prepare
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
@@ -104,6 +137,8 @@ jobs:
       before_script:
         - pip3 install -r requirements.txt
         - cd deploy
+        - make -C images -j 2 pull-base pull-inbox
+        - make -C images -j 2 retag-base retag-inbox
         - make prepare
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
@@ -112,10 +147,20 @@ jobs:
       script:
         - cd tests
         - mvn test -Dtest=RobustnessTests
-
+    #
+    # Retagging the images
+    #
     - stage: images
-      name: "Images Build"
-      script: echo 'Next PR'
+      services: docker
+      name: "Images Retag"
+      before_script:
+        - cd deploy/images
+      script:
+        - make -j 2 pull-base pull-inbox
+        - make retag-base retag-inbox
+        - make -j 2 push-base push-inbox IMGTAG=latest
+        - make clean-tag-base clean-tag-inbox
+
 
 notifications:
   email: false

diff --git a/Dockerfile.base b/Dockerfile.base
@@ -0,0 +1,34 @@
+FROM python:3.6-alpine3.8
+LABEL maintainer "EGA System Developers"
+
+RUN apk add --no-cache \
+    libressl libressl-dev make gcc musl-dev libffi-dev postgresql-libs postgresql-dev git
+
+#################################################
+##
+## Install LocalEGA stuff
+##
+#################################################
+
+ARG LEGA_GID=1000
+RUN addgroup -g ${LEGA_GID} lega && \
+    adduser -D -G lega lega
+
+COPY setup.py /root/LocalEGA/setup.py
+COPY lega /root/LocalEGA/lega
+COPY requirements.txt /root/LocalEGA/requirements.txt
+
+RUN pip install -r /root/LocalEGA/requirements.txt
+RUN pip install /root/LocalEGA && \
+    rm -rf /root/LocalEGA
+
+RUN apk del --no-cache --purge \
+            gcc git make \
+	    postgresql-dev musl-dev libffi-dev libressl-dev \
+    && rm -rf /var/cache/apk/*
+
+# Replaces the need for gosu
+# But instead, we use the 'user:' in the docker-compose file
+# to still leave the door open for debugging
+#
+# USER lega
diff --git a/Dockerfile.inbox b/Dockerfile.inbox
@@ -0,0 +1,18 @@
+FROM egarchive/lega-inbox-os:latest
+LABEL maintainer "EGA System Developers"
+
+#################################################
+##
+## Install the notification system
+##
+#################################################
+
+# These are enough
+RUN pip3.6 install --upgrade pip && \
+    pip3.6 install PyYaml pika==0.12.0 uvloop
+
+COPY setup.py /root/LocalEGA/setup.py
+COPY lega /root/LocalEGA/lega
+
+RUN pip install /root/LocalEGA && \
+    rm -rf /root/LocalEGA
diff --git a/deploy/bootstrap/run.sh b/deploy/bootstrap/run.sh
@@ -340,7 +340,7 @@ cat >> ${PRIVATE}/lega.yml <<EOF  # SFTP inbox
       - CEGA_ENDPOINT_JSON_PREFIX=response.result
     ports:
       - "${DOCKER_PORT_inbox}:9000"
-    image: nbisweden/ega-inbox:latest
+    image: egarchive/lega-inbox:latest
     volumes:
       - ./conf.ini:/etc/ega/conf.ini:ro
       - ../images/inbox/entrypoint.sh:/usr/local/bin/entrypoint.sh
@@ -355,7 +355,7 @@ cat >> ${PRIVATE}/lega.yml <<EOF
     depends_on:
       - db
       - mq
-    image: nbisweden/ega-base:latest
+    image: egarchive/lega-base:latest
     container_name: finalize
     labels:
         lega_label: "finalize"
@@ -365,14 +365,15 @@ cat >> ${PRIVATE}/lega.yml <<EOF
     restart: on-failure:3
     networks:
       - lega
-    entrypoint: ["gosu", "lega", "ega-id-mapper"]
+    user: lega
+    entrypoint: ["ega-finalize"]
 
   # Ingestion Workers
   ingest:
     depends_on:
       - db
       - mq
-    image: nbisweden/ega-base:latest
+    image: egarchive/lega-base:latest
     container_name: ingest
     labels:
         lega_label: "ingest"
@@ -388,7 +389,8 @@ cat >> ${PRIVATE}/lega.yml <<EOF
     restart: on-failure:3
     networks:
       - lega
-    entrypoint: ["gosu", "lega", "ega-ingest"]
+    user: lega
+    entrypoint: ["ega-ingest"]
 
   # Key server
 EOF
@@ -425,7 +427,7 @@ cat >> ${PRIVATE}/lega.yml <<EOF
     container_name: keys
     labels:
         lega_label: "keys"
-    image: nbisweden/ega-base:latest
+    image: egarchive/lega-base:latest
     expose:
       - "8443"
     environment:
@@ -442,7 +444,8 @@ cat >> ${PRIVATE}/lega.yml <<EOF
     restart: on-failure:3
     networks:
       - lega
-    entrypoint: ["gosu","lega","ega-keyserver","--keys","/etc/ega/keys.ini.enc"]
+    user: lega
+    entrypoint: ["ega-keyserver","--keys","/etc/ega/keys.ini.enc"]
 
 EOF
 fi
@@ -457,7 +460,7 @@ cat >> ${PRIVATE}/lega.yml <<EOF
     container_name: verify
     labels:
         lega_label: "verify"
-    image: nbisweden/ega-base:latest
+    image: egarchive/lega-base:latest
     environment:
       - LEGA_PASSWORD=${LEGA_PASSWORD}
       - VAULT_ACCESS_KEY=${S3_ACCESS_KEY}
@@ -470,7 +473,8 @@ cat >> ${PRIVATE}/lega.yml <<EOF
     restart: on-failure:3
     networks:
       - lega
-    entrypoint: ["gosu", "lega", "ega-verify"]
+    user: lega
+    entrypoint: ["ega-verify"]
 
   # Data Out re-encryption service
   res:
@@ -582,7 +586,7 @@ services:
     hostname: cega-users
     ports:
       - "15671:80"
-    image: nbisweden/ega-base:latest
+    image: egarchive/lega-base:latest
     container_name: cega-users
     labels:
         lega_label: "cega-users"
@@ -591,7 +595,7 @@ services:
        - ../../tests/_common/users.json:/cega/users.json
     networks:
       - lega
-    entrypoint: ["python3.6", "/cega/users.py", "0.0.0.0", "80", "/cega/users.json"]
+    entrypoint: ["python", "/cega/users.py", "0.0.0.0", "80", "/cega/users.json"]
 EOF
 
     # The user/password is legatest:legatest