Merge pull request #84 from EGA-archive/c4gh-update

C4gh update

.coveragerc

@@ -10,8 +10,6 @@ omit =
     /usr/*
     # omit this single file
     lega/conf/__init__.py
-    lega/openpgp/constants.py
-    lega/openpgp/__init__.py
     lega/utils/exceptions.py
     lega/utils/logging.py
 

.gitignore

@@ -8,7 +8,7 @@ deploy/bootstrap/settings.rc.private
 tests/**/*.sec
 tests/**/*.debug
 tests/**/*.d
-tests/_common/users.json
+tests/_common/users/
 tests/_common/backup
 tests/_common/mq/*.pem
 

.travis.yml

@@ -20,7 +20,9 @@ jobs:
     - stage: tests
       name: "Unit Tests"
       python: 3.6
-      before_script: pip install tox-travis
+      before_script:
+        - pip install --upgrade pip
+        - pip install tox-travis
       script: tox
     #
     # BATS Tests
@@ -34,7 +36,7 @@ jobs:
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
         - make -C images
-        - make install-bootstrap-packages prepare
+        - sudo make install-bootstrap-packages
         - make bootstrap
         - make up
         - make preflight-check
@@ -50,7 +52,7 @@ jobs:
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
         - make -C images
-        - make install-bootstrap-packages prepare
+        - sudo make install-bootstrap-packages
         - make bootstrap ARGS='--archive-backend posix'
         - make up
         - make preflight-check
@@ -66,7 +68,7 @@ jobs:
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
         - make -C images
-        - make install-bootstrap-packages prepare
+        - sudo make install-bootstrap-packages
         - make bootstrap
         - make up
         - make preflight-check
@@ -82,47 +84,16 @@ jobs:
         - pushd bats-core && git checkout v1.1.0 && sudo ./install.sh /usr/local && popd
         - cd deploy
         - make -C images
-        - make install-bootstrap-packages prepare
+        - sudo make install-bootstrap-packages
         - make bootstrap
         - make up
         - make preflight-check
       script:
         - cd ../tests
         - bats security
-    # #
-    # # Cucumber Tests
-    # #
-    # - stage: tests
-    #   name: "Cucumber Ingestion Tests"
-    #   if: type = pull_request
-    #   before_script:
-    #     - pip3 install -r requirements.txt
-    #     - cd deploy
-    #     - make -C images
-    #     - make prepare
-    #     - make bootstrap ARGS='--inbox mina'
-    #     - make up
-    #     - make preflight-check
-    #   script:
-    #     - cd tests
-    #     - mvn test -Dtest=IngestionTests
-    # - stage: tests
-    #   name: "Cucumber Robustness Tests"
-    #   if: type = pull_request
-    #   before_script:
-    #     - pip3 install -r requirements.txt
-    #     - cd deploy
-    #     - make -C images
-    #     - make prepare
-    #     - make bootstrap ARGS='--inbox mina'
-    #     - make up
-    #     - make preflight-check
-    #   script:
-    #     - cd tests
-    #     - mvn test -Dtest=RobustnessTests
 
 
-notifications:
-  email: false
-  slack:
-    secure: eUyEWWvrFbzW+j+WKIOrHm7zeJ+6+o/WmI5cp1UYsOT9emxGE4kzW057cG9EV+sgKUdoYP1zSfCH0TLSOjY7otyqccqZH5WxDtiBSEXpkA8ID8jzQnX1VZWFn1vK+gWpER87VdLonVGt4db1lqE3Gm/uCbEqzrmfYjE1Hrk4PM8FfLQfD3+YBPUnWGSZKAPmdHAKh7IF9VQ6f1zaspijp/Sxa7Dk9F+Z4o2nsZ1woSyOVAwWLJhkvEafyEFfb/9tPMF1wtoXlLEzV1JDRzyjzbLGXQcpo6+Qx3+v7w6eRbriifOq2tByBfeI+RlWytwOgb+B/mfN0uFPbdg/Bgr//NMDrqwCnFQs7A2Dj287mQZI4YpRvh4Cneu3ReVGQKd9SJq28BliwXBBv3xyeFfGEbBOMNKb0VCsNjRuWITncf/qx3Vxn13VAYxcdA9EZpa1UzT6V94nlbLUq3twFKBJiDmpraYnI+JGFCZ32Xh8bySNqbEBe7TnqAG015c4pKKx++3IQJePfSPbRKzwWNAM5yG7RuVmud5fxfN+KdQz7vKfjOeaHKG4PScfhRT0zthtgmPG+m5eCprIbdFlacU3UyobLtxZd8wI9qJnGGvB3bHOsuaqpS2ymDWbd/n1aeryrcTkS/gPuwMvTs6S32pRf/orKqyLfnSZPeTcOevbzHw=
+# notifications:
+#   email: false
+#   slack:
+#     secure: eUyEWWvrFbzW+j+WKIOrHm7zeJ+6+o/WmI5cp1UYsOT9emxGE4kzW057cG9EV+sgKUdoYP1zSfCH0TLSOjY7otyqccqZH5WxDtiBSEXpkA8ID8jzQnX1VZWFn1vK+gWpER87VdLonVGt4db1lqE3Gm/uCbEqzrmfYjE1Hrk4PM8FfLQfD3+YBPUnWGSZKAPmdHAKh7IF9VQ6f1zaspijp/Sxa7Dk9F+Z4o2nsZ1woSyOVAwWLJhkvEafyEFfb/9tPMF1wtoXlLEzV1JDRzyjzbLGXQcpo6+Qx3+v7w6eRbriifOq2tByBfeI+RlWytwOgb+B/mfN0uFPbdg/Bgr//NMDrqwCnFQs7A2Dj287mQZI4YpRvh4Cneu3ReVGQKd9SJq28BliwXBBv3xyeFfGEbBOMNKb0VCsNjRuWITncf/qx3Vxn13VAYxcdA9EZpa1UzT6V94nlbLUq3twFKBJiDmpraYnI+JGFCZ32Xh8bySNqbEBe7TnqAG015c4pKKx++3IQJePfSPbRKzwWNAM5yG7RuVmud5fxfN+KdQz7vKfjOeaHKG4PScfhRT0zthtgmPG+m5eCprIbdFlacU3UyobLtxZd8wI9qJnGGvB3bHOsuaqpS2ymDWbd/n1aeryrcTkS/gPuwMvTs6S32pRf/orKqyLfnSZPeTcOevbzHw=

Dockerfile

@@ -1,16 +1,23 @@
-FROM python:3.6-alpine3.8 as BUILD
+##########################
+## Build env
+##########################
+FROM python:3.6-alpine3.10 as BUILD
 
 RUN apk add git postgresql-dev gcc musl-dev libffi-dev make gnupg
 
+COPY requirements.txt /root/LocalEGA/requirements.txt
+RUN pip install --upgrade pip && \
+    pip install -r /root/LocalEGA/requirements.txt
+
 COPY setup.py /root/LocalEGA/setup.py
 COPY lega /root/LocalEGA/lega
-COPY requirements.txt /root/LocalEGA/requirements.txt
+RUN pip install /root/LocalEGA
 
-RUN pip install --upgrade pip && \
-    pip install -r /root/LocalEGA/requirements.txt && \
-    pip install /root/LocalEGA
 
-FROM python:3.6-alpine3.8
+##########################
+## Final image
+##########################
+FROM python:3.6-alpine3.10
 
 ARG BUILD_DATE
 ARG SOURCE_COMMIT
@@ -31,8 +38,6 @@ RUN addgroup -g ${LEGA_GID} lega && \
 
 COPY --from=BUILD usr/local/lib/python3.6/ usr/local/lib/python3.6/
 
-COPY --from=BUILD /usr/local/bin/lega-cryptor /usr/local/bin/
-
 COPY --from=BUILD /usr/local/bin/ega-* /usr/local/bin/
 
 RUN mkdir -p /ega/archive && \

deploy/Makefile

@@ -1,5 +1,5 @@
 SHELL := /bin/bash
-
+OPENSSL=openssl
 ARGS=
 
 .PHONY: help bootstrap private network up down clean ps clean-volumes clean-all prepare images install-bootstrap-packages
@@ -48,21 +48,12 @@ preflight-check:
 # - the key server (slow setup with java spring)
 
 
-# Decrypt the encoded data from the repository
-# That is, some users private ssh keys and the database of fake CentralEGA Users
-prepare: ../tests/_common/dummy.sec ../tests/_common/john.sec ../tests/_common/jane.sec ../tests/_common/users.json
-
-%.sec: %.sec.enc
-	openssl enc -aes-256-cbc -d -in $< -out $@ -k ${REPO_PASSWORD}
-	chmod 400 $@
-
-%.json: %.json.enc
-	openssl enc -aes-256-cbc -d -in $< -out $@ -k ${REPO_PASSWORD}
-
 images:
 	make -C images
 
 # That's for Travis
 install-bootstrap-packages:
-	pip3 install git+https://github.com/NBISweden/LocalEGA-cryptor.git
-	pip3 install pgpy pika==1.0.1
+	pip3 install --upgrade pip
+	pip3 install git+https://github.com/EGA-archive/crypt4gh.git
+	pip3 install pika==1.0.1
+	apt-get install expect

deploy/README.md

@@ -8,8 +8,10 @@ You can then [generate the private data](bootstrap), with:
 
 	make bootstrap
 
+This requires `openssl` (>=1.0), `ssh-keygen` (=>6.5), `expect` and [`crypt4gh-keygen`](https://github.com/EGA-archive/crypt4gh).
+
 The command will create a `.env` file and a `private` folder holding
-the necessary data (ie the PGP key, the Main LEGA password, the SSL
+the necessary data (ie the master keypair, the SSL
 certificates for internal communication, passwords, default users,
 etc...)
 
@@ -20,26 +22,15 @@ These networks are reflected in their corresponding YML files
 * `private/cega.yml`
 * `private/lega.yml`
 
-The passwords are in `private/lega/.trace` and the errors (if any) are in `private/.err`.
-
-### Bootstrapping with advanced options
-
-If one would want to make use of the [keyserver from
-ega-data-api](https://github.com/EGA-archive/ega-data-api/tree/master/ega-data-api-key) instead of the [LocalEGA keyserver](https://github.com/NBISweden/LocalEGA/blob/dev/lega/keyserver.py), bootstrap with:
-
-	make "ARGS=--keyserver ega" bootstrap
-
-In order to make use of [Apache Mina Inbox](https://github.com/NBISweden/LocalEGA-inbox) use:
+The passwords are in `private/.trace` and the errors (if any) are in `private/.err`.
 
-	make bootstrap ARGS='--inbox mina'
-
-Multiple bootstrapping options can be used at the same time:
-
-	make bootstrap ARGS='--inbox mina --keyserver ega'
+The test user credentials are found in `private/.users`.
 
 ## Running
 
-	docker-compose up -d
+	make up
+
+This is just a shortcut for `docker-compose up -d`
 
 Use `docker-compose up -d --scale ingest=3 --scale verify=5` instead,
 if you want to start 3 ingestion and 5 verification workers.
@@ -50,17 +41,21 @@ will be created on-the-fly by docker-compose.
 
 ## Stopping
 
-	docker-compose down -v
+	make down
+
+This is just a shortcut for `docker-compose down -v` (removing networks and volumes).
 
 ## Status
 
-	docker-compose ps
+	make down
+
+This is just a shortcut for `docker-compose ps`
 
 ## Cleaning
 
 To clean up everything you can use the following commands
 
-Remove the bootstrap stuff (including network):
+Remove the bootstrap stuff (including private data and certificates):
 
     make clean