Merge pull request #5 from EGA-archive/renaming

Renaming

.travis.yml

@@ -15,12 +15,8 @@ services: docker
 stages:
   - name: unit tests
     if: type IN (push, pull_request)
-  - name: image build
-    if: type IN (pull_request)
   - name: integration tests
     if: type IN (pull_request)
-  - name: tag image
-    if: type IN (push, pull_request)
 
 jobs:
   include:
@@ -30,23 +26,9 @@ jobs:
         - pip install tox-travis
       # command to run tests
       script: tox
-    - stage: image build
-      before_script:
-        - cd docker
-      script:
-        - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USER" --password-stdin
-        - ../extras/travis_check_image_change.sh os
-        # Pulling image for layer caching
-        - docker pull nbisweden/ega-openssh
-        - docker pull nbisweden/ega-base
-        - docker pull nbisweden/ega-inbox
-        - make -C images base inbox
-        - docker push nbisweden/ega-base:"PR${TRAVIS_PULL_REQUEST}"
-        - docker push nbisweden/ega-inbox:"PR${TRAVIS_PULL_REQUEST}"
     - stage: integration tests
       before_script:
-        - cd docker
-        - ../extras/travis_push_docker_hub.sh
+        - cd deploy
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
         - docker network create cega
@@ -57,8 +39,7 @@ jobs:
         - mvn test -Dtest=IngestionTests -B
     - stage: integration tests
       before_script:
-        - cd docker
-        - ../extras/travis_push_docker_hub.sh
+        - cd deploy
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
         - docker network create cega
@@ -69,8 +50,7 @@ jobs:
         - mvn test -Dtest=OutgestionTests -B
     - stage: integration tests
       before_script:
-        - cd docker
-        - ../extras/travis_push_docker_hub.sh
+        - cd deploy
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
         - docker network create cega
@@ -81,8 +61,7 @@ jobs:
         - mvn test -Dtest=RobustnessTests -B
     - stage: integration tests
       before_script:
-        - cd docker
-        - ../extras/travis_push_docker_hub.sh
+        - cd deploy
         - make bootstrap
         - sudo chown -R travis private
         - docker network create cega
@@ -96,8 +75,6 @@ jobs:
         - make -C test
         - sleep 10
         - make -C test check
-    - stage: tag image
-      script: ./extras/travis_push_docker_hub.sh
 
 notifications:
   email: false

README.md

@@ -1,32 +1,33 @@
-# NBIS repository for the Local EGA project
+# Local EGA main repository
 
-[![Codacy Badge](https://api.codacy.com/project/badge/Grade/3dd83b28ec2041889bfb13641da76c5b)](https://www.codacy.com/app/NBIS/LocalEGA?utm_source=github.com&utm_medium=referral&utm_content=NBISweden/LocalEGA&utm_campaign=Badge_Grade)
-[![Build Status](https://travis-ci.org/NBISweden/LocalEGA.svg?branch=dev)](https://travis-ci.org/NBISweden/LocalEGA)
-[![Coverage Status](https://coveralls.io/repos/github/NBISweden/LocalEGA/badge.svg?branch=dev)](https://coveralls.io/github/NBISweden/LocalEGA?branch=dev)
+[![Build Status](https://travis-ci.org/EGA-archive/LocalEGA.svg?branch=dev)](https://travis-ci.org/EGA-archive/LocalEGA)
+[![Documentation Status](https://readthedocs.org/projects/localega/badge/?version=latest)](https://localega.readthedocs.io/en/latest/?badge=latest)
+[![Coverage Status](https://coveralls.io/repos/github/EGA-archive/LocalEGA/badge.svg?branch=master)](https://coveralls.io/github/EGA-archive/LocalEGA?branch=master)
 
 The [code](lega) is written in Python (3.6+).
 
-You can provision and deploy the different components:
+You can provision and deploy the different components, locally, using [docker-compose](deploy).
 
-* locally, using [docker-compose](docker);
-* on an OpenStack cluster, using [terraform](https://github.com/NBISweden/LocalEGA-deploy-terraform);
-* on a Kubernetes/OpenShift cluster, using [kubernetes](https://github.com/NBISweden/LocalEGA-deploy-k8s);
-* on a Docker Swarm cluster, using [Gradle](https://github.com/NBISweden/LocalEGA-deploy-swarm).
+Other provisioning methods are provided by our partners:
+
+* on an [OpenStack cluster](https://github.com/NBISweden/LocalEGA-deploy-terraform), using `terraform`;
+* on a [Kubernetes/OpenShift cluster](https://github.com/NBISweden/LocalEGA-deploy-k8s), using `kubernetes`;
+* on a [Docker Swarm cluster](https://github.com/NBISweden/LocalEGA-deploy-swarm), using `gradle`.
 
 # Architecture
 
-LocalEGA is divided into several components, whether as docker
-containers or as virtual machines.
+LocalEGA is divided into several components, as docker containers.
 
 | Components  | Role |
 |-------------|------|
-| db          | A Postgres database with appropriate schema |
-| mq          | A RabbitMQ message broker with appropriate accounts, exchanges, queues and bindings |
-| inbox       | SFTP server, acting as a dropbox, where user credentials come from CentralEGA |
-| keyserver   | Handles the encryption/decryption keys |
-| ingesters   | Split the Crypt4GH header and move the remainder to the storage backend. No cryptographic task, nor connection to the keyserver. |
-| verifiers   | Connect to the keyserver (via SSL) and decrypt the stored files and checksum them against their embedded checksum. |
+| db          | A Postgres database with appropriate schemas and isolations |
+| mq          | A (local) RabbitMQ message broker with appropriate accounts, exchanges, queues and bindings, connected to the CentralEGA counter-part. |
+| inbox       | SFTP server, acting as a dropbox, where user credentials are fetched from CentralEGA |
+| ingesters   | Split the Crypt4GH header and move the remainder to the storage backend. No cryptographic task, nor access to the decryption keys. |
+| verifiers   | Decrypt the stored files and checksum them against their embedded checksum. |
 | vault       | Storage backend: as a regular file system or as a S3 object store. |
-| ID mapper   | Handles the so-called _Stable ID_ filename mappings from CentralEGA. |
+| finalizers  | Handle the so-called _Stable ID_ filename mappings from CentralEGA. |
+| outgesters  | Front-facing checks for download permissions. |
+| streamers   | Fetch the files from the vault and re-encrypt its header for the given requester. |
 
 Find the [LocalEGA documentation](http://localega.readthedocs.io) hosted on [ReadTheDocs.org](https://readthedocs.org/).

docker/Makefile → deploy/Makefile

@@ -16,7 +16,7 @@ private/cega.yml private/lega.yml private bootstrap:
 		    -v ${PWD}/../extras/generate_pgp_key.py:/tmp/generate_pgp_key.py \
 		    -v ${PWD}/../extras/rabbitmq_hash.py:/tmp/rabbitmq_hash.py \
 		    --entrypoint /ega/bootstrap/boot.sh \
-		    nbisweden/ega-base:dev ${ARGS}
+		    nbisweden/ega-base:latest ${ARGS}
 
 network:
 	@docker network inspect cega &>/dev/null || docker network create cega &>/dev/null

docker/bootstrap/cega.sh → deploy/bootstrap/cega.sh

@@ -173,7 +173,7 @@ services:
 
   cega-users:
     env_file: cega/env
-    image: nbisweden/ega-base:dev
+    image: nbisweden/ega-base:latest
     hostname: cega-users
     container_name: cega-users
     labels:
@@ -203,7 +203,7 @@ cat >> ${PRIVATE}/cega.yml <<EOF
     #  - "8761:8761"
     expose:
       - 8761
-    image: nbisweden/ega-base:dev
+    image: nbisweden/ega-base:latest
     container_name: cega-eureka
     labels:
         lega_label: "cega-eureka"

docker/bootstrap/lega.sh → deploy/bootstrap/lega.sh

@@ -84,7 +84,7 @@ chroot_sessions = True
 
 [vault]
 driver = S3Storage
-url = http://s3:9000
+url = http://vault:9000
 access_key = ${S3_ACCESS_KEY}
 secret_key = ${S3_SECRET_KEY}
 #region = lega
@@ -226,7 +226,7 @@ cat >> ${PRIVATE}/lega.yml <<EOF  # SFTP inbox
       - CEGA_ENDPOINT_JSON_PREFIX=response.result
     ports:
       - "${DOCKER_PORT_inbox}:9000"
-    image: nbisweden/ega-inbox:dev
+    image: nbisweden/ega-inbox:latest
     volumes:
       - ./lega/conf.ini:/etc/ega/conf.ini:ro
       - inbox:/ega/inbox
@@ -235,14 +235,14 @@ fi
 
 cat >> ${PRIVATE}/lega.yml <<EOF
   # Stable ID mapper
-  id-mapper:
+  finalize:
     depends_on:
       - db
       - mq
-    image: nbisweden/ega-base:dev
-    container_name: id-mapper
+    image: nbisweden/ega-base:latest
+    container_name: finalize
     labels:
-        lega_label: "id-mapper"
+        lega_label: "finalize"
     volumes:
        - ./lega/conf.ini:/etc/ega/conf.ini:ro
     restart: on-failure:3
@@ -255,7 +255,7 @@ cat >> ${PRIVATE}/lega.yml <<EOF
     depends_on:
       - db
       - mq
-    image: nbisweden/ega-base:dev
+    image: nbisweden/ega-base:latest
     container_name: ingest
     labels:
         lega_label: "ingest"
@@ -307,7 +307,7 @@ cat >> ${PRIVATE}/lega.yml <<EOF
     container_name: keys
     labels:
         lega_label: "keys"
-    image: nbisweden/ega-base:dev
+    image: nbisweden/ega-base:latest
     expose:
       - "8443"
     environment:
@@ -341,7 +341,7 @@ cat >> ${PRIVATE}/lega.yml <<EOF
     container_name: verify
     labels:
         lega_label: "verify"
-    image: nbisweden/ega-base:dev
+    image: nbisweden/ega-base:latest
     environment:
       - LEGA_PASSWORD=${LEGA_PASSWORD}
       - S3_ACCESS_KEY=${S3_ACCESS_KEY}
@@ -358,7 +358,7 @@ cat >> ${PRIVATE}/lega.yml <<EOF
   # Data Out re-encryption service
   res:
     depends_on:
-      - s3
+      - vault
       - keys
     hostname: res
     container_name: res
@@ -379,26 +379,26 @@ cat >> ${PRIVATE}/lega.yml <<EOF
       - EGA_SHAREDPASS_PATH=/etc/ega/pgp/ega.shared.pass
       - EGA_EBI_AWS_ACCESS_KEY=${S3_ACCESS_KEY}
       - EGA_EBI_AWS_ACCESS_SECRET=${S3_SECRET_KEY}
-      - EGA_EBI_AWS_ENDPOINT_URL=http://s3:${DOCKER_PORT_s3}
+      - EGA_EBI_AWS_ENDPOINT_URL=http://vault:${DOCKER_PORT_s3}
       - EGA_EBI_AWS_ENDPOINT_REGION=
     volumes:
       - ./lega/pgp/ega.shared.pass:/etc/ega/pgp/ega.shared.pass:ro
     restart: on-failure:3
     networks:
       - lega
 
-  # S3
-  s3:
-    hostname: s3
-    container_name: s3
+  # Storage backend: S3
+  vault:
+    hostname: vault
+    container_name: vault
     labels:
-        lega_label: "s3"
+        lega_label: "vault"
     image: minio/minio
     environment:
       - MINIO_ACCESS_KEY=${S3_ACCESS_KEY}
       - MINIO_SECRET_KEY=${S3_SECRET_KEY}
     volumes:
-      - s3:/data
+      - vault:/data
     restart: on-failure:3
     networks:
       - lega
@@ -410,7 +410,7 @@ cat >> ${PRIVATE}/lega.yml <<EOF
 volumes:
   db:
   inbox:
-  s3:
+  vault:
 EOF
 
 #########################################################################

docker/test/Makefile → deploy/test/Makefile

@@ -19,7 +19,7 @@ get_shasum =                                          \
 
 ##############################
 
-DOCKER_PATH=$(MAIN_REPO)/docker
+DOCKER_PATH=$(MAIN_REPO)/deploy
 CEGA_USERS=$(DOCKER_PATH)/private/cega/users
 
 INSTANCE_PORT=$(shell awk -F= '/DOCKER_PORT_inbox/ {print $$2}' $(DOCKER_PATH)/bootstrap/settings.rc)

docker/tests/pom.xml → deploy/tests/pom.xml

@@ -62,7 +62,7 @@
         <dependency>
             <groupId>no.uio.ifi</groupId>
             <artifactId>crypt4gh</artifactId>
-            <version>1.0.0</version>
+            <version>1.2.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>

docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Uploading.java → deploy/tests/src/test/java/se/nbis/lega/cucumber/steps/Uploading.java

@@ -3,7 +3,7 @@
 import cucumber.api.java8.En;
 import lombok.extern.slf4j.Slf4j;
 import net.schmizz.sshj.sftp.RemoteResourceInfo;
-import no.ifi.uio.crypt4gh.stream.Crypt4GHOutputStream;
+import no.uio.ifi.crypt4gh.stream.Crypt4GHOutputStream;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;

docker/tests/src/test/resources/config.properties → deploy/tests/src/test/resources/config.properties

@@ -11,5 +11,5 @@ container.label.inbox = inbox
 container.label.ingest = ingest
 container.label.keys = keys
 container.label.mq = mq
-container.label.s3 = s3
+container.label.s3 = vault
 container.label.verify = verify

docs/code.rst

@@ -14,7 +14,7 @@ Python Modules
     lega.ingest
     lega.verify
     lega.keyserver
-    lega.mapper
+    lega.finalize
 
 *******************
 Inbox notifications
@@ -31,7 +31,7 @@ Ingestion Worker
    :members:
 .. automodule:: lega.verify
    :members:
-.. automodule:: lega.mapper
+.. automodule:: lega.finalize
    :members:
 
 

setup.py

@@ -26,7 +26,7 @@
               'ega-verify = lega.verify:main',
               'ega-keyserver = lega.keyserver:main',
               'ega-notifications = lega.notifications:main',
-              'ega-id-mapper = lega.mapper:main',
+              'ega-finalize = lega.finalize:main',
               'ega-conf = lega.conf.__main__:main',
           ]
       },

tests/test_mapper.py → tests/unit/test_finalize.py

@@ -1,12 +1,12 @@
 import unittest
-from lega.mapper import main, work
+from lega.finalize import main, work
 from unittest import mock
 
 
-class testMapper(unittest.TestCase):
-    """Mapper.
+class testFinalize(unittest.TestCase):
+    """Finalize.
 
-    Testing mapper functionalities.
+    Testing Finalizer functionalities.
     """
 
     def setUp(self):
@@ -17,18 +17,18 @@ def tearDown(self):
         """Remove anything that was setup."""
         pass
 
-    @mock.patch('lega.mapper.db')
+    @mock.patch('lega.finalize.db')
     def test_work(self, mock_db):
-        """Test mapper worker, should insert into database."""
+        """Test finalize worker, should insert into database."""
         # mock_db.set_stable_id.return_value = mock.Mock()
         data = {'stable_id': '1', 'file_id': '123'}
         work(data)
         mock_db.set_stable_id.assert_called_with('123', '1')
 
-    @mock.patch('lega.mapper.get_connection')
-    @mock.patch('lega.mapper.consume')
+    @mock.patch('lega.finalize.get_connection')
+    @mock.patch('lega.finalize.consume')
     def test_main(self, mock_consume, mock_connection):
-        """Test main mapper, by mocking cosume call."""
+        """Test main finalize, by mocking cosume call."""
         mock_consume.return_value = mock.MagicMock()
         main()
         mock_consume.assert_called()

tox.ini

@@ -4,7 +4,7 @@ envlist = unit_tests, flake8
 [flake8]
 ignore = E226,D203,D212,D213,D404,D100,D104
 exclude =
-    docker,
+    deploy,
     extras,
     .tox
 max-line-length = 160
@@ -19,10 +19,10 @@ commands = flake8 .
 passenv = TRAVIS TRAVIS_*
 deps =
     -rrequirements.txt
-    -rtests/requirements.txt
+    -rtests/unit/requirements.txt
 # Stop after first failure
-commands = pytest -x --cov=lega tests/
-           python {toxinidir}/tests/coveralls.py
+commands = pytest -x --cov=lega tests/unit
+           python {toxinidir}/tests/unit/coveralls.py
 
 [travis]
 unignore_outcomes = True