Merge pull request #42 from EGA-archive/feature/inbox-events

Handling inbox files events
EGA-archive · Mar 19, 2019 · ab597f0 · ab597f0
2 parents e6aa534 + c1bf8ea
commit ab597f0
Show file tree

Hide file tree

Showing 24 changed files with 81 additions and 723 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -34,10 +34,10 @@ jobs:
         - pip install tox-travis
         - cd deploy/images
         # Pulling image for layer caching
-        - make -j 2 pull-base pull-inbox IMGTAG=latest
+        - make -j 2 pull-base IMGTAG=latest
       script:
-        - make -j 4 base inbox unit-tests
-        - make -j 2 push-base push-inbox
+        - make -j 4 base unit-tests
+        - make -j 2 push-base
     #
     # BATS Tests
     #
@@ -49,8 +49,8 @@ jobs:
         - git clone https://github.com/bats-core/bats-core.git
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
-        - make -C images -j 2 pull-base pull-inbox
-        - make -C images -j 2 retag-base retag-inbox
+        - make -C images -j 2 pull-base
+        - make -C images -j 2 retag-base
         - make prepare
         - make bootstrap ARGS="--keyserver ega"
         - sudo chown -R travis private
@@ -67,8 +67,8 @@ jobs:
         - git clone https://github.com/bats-core/bats-core.git
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
-        - make -C images -j 2 pull-base pull-inbox
-        - make -C images -j 2 retag-base retag-inbox
+        - make -C images -j 2 pull-base
+        - make -C images -j 2 retag-base
         - make prepare
         - make bootstrap ARGS="--keyserver ega"
         - sudo chown -R travis private
@@ -89,8 +89,8 @@ jobs:
         - git clone https://github.com/bats-core/bats-core.git
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
-        - make -C images -j 2 pull-base pull-inbox
-        - make -C images -j 2 retag-base retag-inbox
+        - make -C images -j 2 pull-base
+        - make -C images -j 2 retag-base
         - make prepare
         - make bootstrap ARGS="--keyserver ega"
         - sudo chown -R travis private
@@ -107,8 +107,8 @@ jobs:
       before_script:
         - pip3 install -r requirements.txt
         - cd deploy
-        - make -C images -j 2 pull-base pull-inbox
-        - make -C images -j 2 retag-base retag-inbox
+        - make -C images -j 2 pull-base
+        - make -C images -j 2 retag-base
         - make prepare
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
@@ -122,8 +122,8 @@ jobs:
       before_script:
         - pip3 install -r requirements.txt
         - cd deploy
-        - make -C images -j 2 pull-base pull-inbox
-        - make -C images -j 2 retag-base retag-inbox
+        - make -C images -j 2 pull-base
+        - make -C images -j 2 retag-base
         - make prepare
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
@@ -137,8 +137,8 @@ jobs:
       before_script:
         - pip3 install -r requirements.txt
         - cd deploy
-        - make -C images -j 2 pull-base pull-inbox
-        - make -C images -j 2 retag-base retag-inbox
+        - make -C images -j 2 pull-base
+        - make -C images -j 2 retag-base
         - make prepare
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
@@ -156,10 +156,10 @@ jobs:
       before_script:
         - cd deploy/images
       script:
-        - make -j 2 pull-base pull-inbox
-        - make retag-base retag-inbox
-        - make -j 2 push-base push-inbox IMGTAG=latest
-        - make clean-tag-base clean-tag-inbox
+        - make -j 2 pull-base
+        - make retag-base
+        - make -j 2 push-base IMGTAG=latest
+        - make clean-tag-base
 
 
 notifications:

diff --git a/Dockerfile.base b/Dockerfile.base
@@ -3,6 +3,7 @@ LABEL maintainer "EGA System Developers"
 
 RUN apk add --no-cache \
     libressl libressl-dev make gcc musl-dev libffi-dev postgresql-libs postgresql-dev git
+RUN pip install --upgrade pip
 
 #################################################
 ##

diff --git a/Dockerfile.inbox b/Dockerfile.inbox
diff --git a/deploy/bootstrap/run.sh b/deploy/bootstrap/run.sh
@@ -332,18 +332,22 @@ cat >> ${PRIVATE}/lega.yml <<EOF
       - inbox:/ega/inbox
 EOF
 else
+# SSL support is temporarily off
 cat >> ${PRIVATE}/lega.yml <<EOF  # SFTP inbox
     environment:
       - CEGA_ENDPOINT=${CEGA_USERS_ENDPOINT}
       - CEGA_ENDPOINT_CREDS=${CEGA_USERS_CREDS}
       - CEGA_ENDPOINT_JSON_PREFIX=response.result
+      - MQ_CONNECTION=amqp://guest:guest@mq:5672/%2F
+      - MQ_EXCHANGE=cega
+      - MQ_ROUTING_KEY=files.inbox
     ports:
       - "${DOCKER_PORT_inbox}:9000"
-    image: egarchive/lega-inbox:latest
+    image: egarchive/lega-inbox:stable
     volumes:
       - ./conf.ini:/etc/ega/conf.ini:ro
-      - ../images/inbox/entrypoint.sh:/usr/local/bin/entrypoint.sh
       - inbox:/ega/inbox
+
 EOF
 fi
 

diff --git a/deploy/images/Makefile b/deploy/images/Makefile
@@ -15,11 +15,11 @@ LEGA_GID=1000
 # Must find better, but working so far
 MAIN_REPO := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../..)
 
-.PHONY: all erase clean purge base inbox push
+.PHONY: all erase clean purge base push
 
-all: base inbox
+all: base
 
-base inbox:
+base:
 	cd $(MAIN_REPO) && \
 	docker build -f Dockerfile.$@ \
 	       --build-arg LEGA_GID=$(LEGA_GID) \
@@ -40,11 +40,9 @@ endef
 
 clean:
 	@$(call clean_tag,$(TARGET_PREFIX)base, $(TAG))
-	@$(call clean_tag,$(TARGET_PREFIX)inbox, $(TAG))
 
 erase:
 	@$(call remove_dangling,$(TARGET_PREFIX)base)
-	@$(call remove_dangling,$(TARGET_PREFIX)inbox)
 
 purge:
 	@$(call remove_dangling,)
@@ -66,14 +64,5 @@ retag-%:
 clean-tag-%:
 	@echo "No way to remove $(TARGET_PREFIX)$(@:clean-tag-%=%):$(TAG) from docker hub at the moment"
 
-
-# Create the inbox base OS (here for convenience)
-inbox-os:
-	docker build \
-	       --build-arg LEGA_GID=$(LEGA_GID) \
-	       --cache-from $(TARGET_PREFIX)$@:latest \
-               --tag $(TARGET_PREFIX)$@:latest \
-               inbox
-
 unit-tests:
 	cd ../.. && tox
diff --git a/deploy/images/README.md b/deploy/images/README.md
@@ -4,42 +4,25 @@
 
 However, we created a Makefile to simplify the building process.
 
-In the current folder, type `make` and the images are created in order.
+In the current folder, the images are created by executing:
 
-It takes some time.
-
-Later on, if the `nbisweden/ega-*` do not need to be recreated, one can type `make all`.
-
-A typical build goes as follows:
-
-	make base
+	make
 
-## Results
-
-`rabbitmq:3.6.14-management`, `postgres:9.6`, `centos:7.5.1804` are pulled from the main Docker hub.
-
-The following images are created locally:
+It takes some time.
 
-| Repository | Tag      | Role |
-|------------|:--------:|------|
-| `nbisweden/ega-os`   | `latest` | CentOS based image with necessary packages for running LocalEGA. |
-| `nbisweden/ega-inbox`  | `<HEAD commit>`, `latest` or `dev`  | SFTP server on top of `nbisweden/ega-base` and `nbisweden/ega-openssh` |
-| `nbisweden/ega-base`   | `<HEAD commit>`, `latest` or `dev` | Base Image for all services includes `python 3.6` |
+The result is an image, named `egarchive/lega-base`, and containing `python 3.6` and the LocalEGA services.
 
-`nbisweden/ega-inbox` is dependent on the OpenSSH image that is built in the https://github.com/NBISweden/LocalEGA-auth
+## Dependencies
 
-| Repository | Tag      | Role |
-|------------|:--------:|------|
-| `nbisweden/ega-openssh`   | `latest` | OpenSSH SFTP server version `7.7p1` on top of `nbisweden/ega-base` patched in order to be used by `nbisweden/ega-inbox` |
+`rabbitmq:3.6.14-management`, `postgres:9.6`, `python:3.6-alpine3.8` are pulled from the main Docker hub.
 
-We also use 2 stubbing services in order to fake the necessary Central EGA components
+The [`egarchive/lega-inbox`](https://github.com/EGA-archive/LocalEGA-inbox) is a LocalEGA inbox, fetching user credentials from CentralEGA and sending file events notifications to the configured message broker. It is based on OpenSSH SFTP server version `7.8p1` 
 
-| Repository | Tag      | Role |
-|------------|:--------:|------|
-| `cega-users` | `<HEAD commit>` , `latest` or `dev` | Sets up a postgres database with appropriate tables, on top of `nbisweden/ega-base` |
-| `cega-mq` | `rabbitmq:3.6.14-management` | Sets up a RabbitMQ message broker with appropriate accounts, exchanges, queues and bindings |
-| `cega-eureka` | `<HEAD commit>`, `latest` or `dev` | Sets up a fake Eureka service discovery server in order to make the LocalEGA Keyserver register, on top of `nbisweden/ega-base` |
+## Testing
 
-## Logging
+We use 2 stubbing services in order to fake the necessary Central EGA components (mostly for local or Travis tests).
 
-ELK stack can be added as a logging solution using the `elasticsearch-oss` `logstash-oss` and `kibana-oss`. This requires adding the images to the `docker-compose` YML.
+| Repository   | Role |
+|--------------|------|
+| `cega-users` | Sets up a small list of test users, on top of `egarchive/lega-base` |
+| `cega-mq`    | Sets up a RabbitMQ message broker with appropriate accounts, exchanges, queues and bindings |