Merge pull request #85 from EGA-archive/cleanup

Merging to move forward.

.dockerignore

@@ -0,0 +1,18 @@
+.coverage
+.coveragerc
+.DS_Store
+.git
+.github
+.gitignore
+.tox
+CONTRIBUTING.md
+deploy
+Dockerfile
+docs
+lega.egg-info
+LICENSE
+README.md
+readthedocs.yml
+tests
+tmp
+tox.ini

.github/workflows/testsuite.yml

@@ -0,0 +1,63 @@
+name: "Testsuite"
+
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - 'tests/*'
+
+jobs:
+  testsuite:
+    strategy:
+      max-parallel: 4
+      matrix:
+        # os: [ubuntu-latest, macOS-latest]
+        os: [ubuntu-latest]
+        bootstrap: ['', '--archive-backend posix']
+        # bats: ['integration', 'security', 'robustness']
+        bats: ['integration']
+
+    runs-on: ${{ matrix.os }}
+
+    name: |
+      ${{ matrix.bats }} tests (bootstrap options: '${{ matrix.bootstrap }}')
+
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@master
+      with:
+        python-version: 3.6
+    - name: (re)Build docker image
+      run: |
+        docker build -f Dockerfile \
+                     --build-arg LEGA_GID=1000 \
+                     --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \
+                     --build-arg SOURCE_COMMIT=$(git rev-parse --short HEAD) \
+                     --tag egarchive/lega-base:latest \
+                     .
+    - name: Install required tools
+      run: |
+        git clone https://github.com/bats-core/bats-core.git
+        pushd bats-core
+        git checkout v1.1.0
+        sudo ./install.sh /usr/local
+        popd
+        pip install git+https://github.com/EGA-archive/crypt4gh.git
+        pip install pika==1.0.1
+        sudo apt-get install expect
+    - name: Bootstrap a LocalEGA (connected to a fake CentralEGA) with '${{ matrix.bootstrap }}'
+      run: |
+        pushd deploy
+        bootstrap/run.sh ${{ matrix.bootstrap }} || { cat private/.err; exit 1; }
+        docker-compose up -d
+        docker-compose ps
+        sleep 80
+        popd
+    - name: Running ${{ matrix.bats }} testsuite
+      run: |
+        bats tests/${{ matrix.bats }}
+
+

.github/workflows/unit.yml

@@ -0,0 +1,39 @@
+name: "Unit Tests"
+
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - 'tests/*'
+
+jobs:
+  unit_tests:
+    strategy:
+      max-parallel: 4
+      matrix:
+        #os: [ubuntu-latest, macOS-latest]
+        os: [ubuntu-latest]
+        python-version: [3.6, 3.7, 3.8]
+
+    runs-on: ${{ matrix.os }}
+
+    name: Unit tests with Python ${{ matrix.python-version }}
+
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@master
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install required tools
+      run: |
+        sudo apt-get install libpq-dev
+        pip install --upgrade pip
+        pip install tox
+    - name: tox
+      run: |
+        tox
+
+

Dockerfile

@@ -19,14 +19,9 @@ RUN pip install /root/LocalEGA
 ##########################
 FROM python:3.6-alpine3.10
 
-ARG BUILD_DATE
-ARG SOURCE_COMMIT
-
 LABEL maintainer "EGA System Developers"
 LABEL org.label-schema.schema-version="1.0"
-LABEL org.label-schema.build-date=$BUILD_DATE
 LABEL org.label-schema.vcs-url="https://github.com/EGA-archive/LocalEGA"
-LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT
 
 
 RUN apk add --no-cache --update libressl postgresql-libs

README.md

@@ -1,9 +1,15 @@
 # Local EGA main repository
 
-[![Build Status](https://travis-ci.org/EGA-archive/LocalEGA.svg?branch=master)](https://travis-ci.org/EGA-archive/LocalEGA)
+
+[![Unit Tests](https://github.com/EGA-archive/LocalEGA/workflows/Unit%20Tests/badge.svg)](https://github.com/EGA-archive/LocalEGA/actions)
+[![Testsuite](https://github.com/EGA-archive/LocalEGA/workflows/Testsuite/badge.svg)](https://github.com/EGA-archive/LocalEGA/actions)
 [![Documentation Status](https://readthedocs.org/projects/localega/badge/?version=latest)](https://localega.readthedocs.io/en/latest/?badge=latest)
 [![Coverage Status](https://coveralls.io/repos/github/EGA-archive/LocalEGA/badge.svg?branch=master)](https://coveralls.io/github/EGA-archive/LocalEGA?branch=master)
 
+[![Docker image](https://github.com/EGA-archive/LocalEGA/workflows/Publish%20to%20Docker%20Hub/badge.svg)](https://hub.docker.com/r/egarchive/lega-base)
+
+
+
 The [code](lega) is written in Python (3.6+).
 
 You can provision and deploy the different components, locally, using [docker-compose](deploy).

deploy/Makefile

@@ -1,8 +1,9 @@
 SHELL := /bin/bash
 OPENSSL=openssl
 ARGS=
+SELECTED=
 
-.PHONY: help bootstrap private network up down clean ps clean-volumes clean-all prepare images install-bootstrap-packages
+.PHONY: help bootstrap private network up down clean ps clean-volumes clean-all erase purge
 
 help:
 	@echo "Usage: make <target>\n"
@@ -15,7 +16,7 @@ bootstrap-dev: export DEPLOY_DEV=yes
 
 
 up: .env private/lega.yml
-	@docker-compose up -d ${OMIT}
+	@docker-compose up -d ${SELECTED}
 
 clean-volumes:
 	docker volume rm lega_db lega_inbox lega_s3
@@ -44,16 +45,36 @@ preflight-check:
 #        egarchive/lega-base /ensure_system_up.sh
 
 # For the moment, we simply do sleep 60, because we need
-# - the rabbitmq shovel to CentralEGA (the federated queue can be late, it doesn't hurt)
-# - the key server (slow setup with java spring)
+# the rabbitmq shovel to CentralEGA (the federated queue can be late, it doesn't hurt)
 
 
-images:
-	make -C images
 
-# That's for Travis
-install-bootstrap-packages:
-	pip3 install --upgrade pip
-	pip3 install git+https://github.com/EGA-archive/crypt4gh.git
-	pip3 install pika==1.0.1
-	apt-get install expect
+####################################################
+## Base Image
+####################################################
+
+# Must find better, but working so far
+MAIN_REPO := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/..)
+
+IMAGE_ARGS=
+# eg --no-cache
+
+image:
+	cd $(MAIN_REPO) && \
+	docker build -f Dockerfile $(IMAGE_ARGS) \
+		   --build-arg LEGA_GID=1000 \
+		   --build-arg BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ') \
+		   --build-arg SOURCE_COMMIT=$(shell git rev-parse --short HEAD) \
+		   --tag egarchive/lega-base:latest \
+		   .
+
+define remove_dangling
+	docker images $(1) -f "dangling=true" -q | uniq | while read n; do docker rmi -f $$n; done
+endef
+
+erase:
+	@$(call remove_dangling,egarchive/lega-base)
+
+purge:
+	@$(call remove_dangling,)
+

deploy/README.md

@@ -2,13 +2,11 @@
 
 ## Bootstrap
 
-First [create the EGA docker images](images) beforehand, with `make -C images`.
-
 You can then [generate the private data](bootstrap), with:
 
 	make bootstrap
 
-This requires `openssl` (>=1.0), `ssh-keygen` (=>6.5), `expect` and [`crypt4gh-keygen`](https://github.com/EGA-archive/crypt4gh).
+This requires `openssl` (>=1.1), `ssh-keygen` (=>6.5), `expect` and [`crypt4gh-keygen`](https://github.com/EGA-archive/crypt4gh).
 
 The command will create a `.env` file and a `private` folder holding
 the necessary data (ie the master keypair, the SSL
@@ -47,7 +45,7 @@ This is just a shortcut for `docker-compose down -v` (removing networks and volu
 
 ## Status
 
-	make down
+	make ps
 
 This is just a shortcut for `docker-compose ps`
 
@@ -66,3 +64,34 @@ Remove the volumes:
 Remove everything:
 
     make clean-all
+
+
+----
+
+# LocalEGA docker image
+
+Create the base image by executing:
+
+	make image
+
+It takes some time. The result is an image, named `egarchive/lega-base`, and containing `python 3.6` and the LocalEGA services.
+
+The following images are pulled from Docker Hub:
+
+* `egarchive/lega-mq` (based on `rabbitmq:3.6.14-management`)
+* `egarchive/lega-db` (based on `postgres:11.2`)
+* `egarchive/lega-inbox` (based on OpenSSH version 7.8p1 and CentOS7)
+* `python:3.6-alpine3.10` 
+
+The [`egarchive/lega-inbox`](https://github.com/EGA-archive/LocalEGA-inbox) is an inbox, fetching user credentials from CentralEGA and sending file events notifications to the configured message broker. It is based on OpenSSH SFTP server version `7.8p1` 
+
+----
+
+# Fake Central EGA
+
+We use 2 stubbing services in order to fake the necessary Central EGA components (mostly for local or Travis tests).
+
+| Container    | Role |
+|--------------|------|
+| `cega-users` | Sets up a small list of test users |
+| `cega-mq`    | Sets up a RabbitMQ message broker with appropriate accounts, exchanges, queues and bindings |