Merge 6634124 into 8737bf1

.travis.yml

@@ -32,7 +32,7 @@ jobs:
       script: tox
     - stage: image build
       before_script:
-        - cd docker
+        - cd deploy
       script:
         - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USER" --password-stdin
         - ../extras/travis_check_image_change.sh os
@@ -45,7 +45,7 @@ jobs:
         - docker push nbisweden/ega-inbox:"PR${TRAVIS_PULL_REQUEST}"
     - stage: integration tests
       before_script:
-        - cd docker
+        - cd deploy
         - ../extras/travis_push_docker_hub.sh
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
@@ -57,7 +57,7 @@ jobs:
         - mvn test -Dtest=IngestionTests -B
     - stage: integration tests
       before_script:
-        - cd docker
+        - cd deploy
         - ../extras/travis_push_docker_hub.sh
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
@@ -69,7 +69,7 @@ jobs:
         - mvn test -Dtest=OutgestionTests -B
     - stage: integration tests
       before_script:
-        - cd docker
+        - cd deploy
         - ../extras/travis_push_docker_hub.sh
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
@@ -79,23 +79,23 @@ jobs:
       script:
         - cd tests
         - mvn test -Dtest=RobustnessTests -B
-    - stage: integration tests
-      before_script:
-        - cd docker
-        - ../extras/travis_push_docker_hub.sh
-        - make bootstrap
-        - sudo chown -R travis private
-        - docker network create cega
-        - docker-compose up -d #--scale ingest=3 --scale verify=5
-        - docker-compose ps
-      script:
-        - sleep 5
-        - pip install -r https://raw.githubusercontent.com/NBISweden/LocalEGA-cryptor/master/requirements.txt
-        - pip install git+https://github.com/NBISweden/LocalEGA-cryptor.git
-        - pip install pika
-        - make -C test
-        - sleep 10
-        - make -C test check
+    # - stage: integration tests
+    #   before_script:
+    #     - cd deploy
+    #     - ../extras/travis_push_docker_hub.sh
+    #     - make bootstrap
+    #     - sudo chown -R travis private
+    #     - docker network create cega
+    #     - docker-compose up -d #--scale ingest=3 --scale verify=5
+    #     - docker-compose ps
+    #   script:
+    #     - sleep 5
+    #     - pip install -r https://raw.githubusercontent.com/NBISweden/LocalEGA-cryptor/master/requirements.txt
+    #     - pip install git+https://github.com/NBISweden/LocalEGA-cryptor.git
+    #     - pip install pika
+    #     - make -C test
+    #     - sleep 10
+    #     - make -C test check
     - stage: tag image
       script: ./extras/travis_push_docker_hub.sh
 

README.md

@@ -27,6 +27,6 @@ containers or as virtual machines.
 | ingesters   | Split the Crypt4GH header and move the remainder to the storage backend. No cryptographic task, nor connection to the keyserver. |
 | verifiers   | Connect to the keyserver (via SSL) and decrypt the stored files and checksum them against their embedded checksum. |
 | vault       | Storage backend: as a regular file system or as a S3 object store. |
-| ID mapper   | Handles the so-called _Stable ID_ filename mappings from CentralEGA. |
+| Finalize    | Handles the so-called _Stable ID_ filename mappings from CentralEGA. |
 
 Find the [LocalEGA documentation](http://localega.readthedocs.io) hosted on [ReadTheDocs.org](https://readthedocs.org/).

docker/Makefile → deploy/Makefile

@@ -7,8 +7,8 @@ help:
 	@echo "where <target> is: 'bootstrap', 'up' 'ps', 'down', 'network' or 'clean'\n"
 
 # If DEPLOY_DEV is yes, we use dummy passwords
-bootstrap: ENVS=--env DEPLOY_DEV=yes
-private/cega.yml private/lega.yml private bootstrap:
+bootstrap-dev: ENVS=--env DEPLOY_DEV=yes
+private/cega.yml private/lega.yml private bootstrap bootstrap-dev:
 	@docker run --rm -it \
 		    -v /dev/urandom:/dev/random \
                     ${ENVS} \

docker/bootstrap/defs.sh → deploy/bootstrap/defs.sh

@@ -51,7 +51,13 @@ function rm_politely {
 }
 
 function generate_password {
+    local force=${2:-no}
+    if [[ "${force}" != "force" ]] && [[ "${DEPLOY_DEV}" = "yes" ]]; then
+	echo "dummy"
+	return 0 # early return
+    fi
+    # Otherwise
     local size=${1:-16} # defaults to 16 characters
-    python3.6 -c "import secrets,string;print(''.join(secrets.choice(string.ascii_letters + string.digits) for i in range(${size})))"
+    ${PYTHON:-python3.6} -c "import secrets,string;print(''.join(secrets.choice(string.ascii_letters + string.digits) for i in range(${size})))"
 }
 

docker/bootstrap/lega.sh → deploy/bootstrap/lega.sh

@@ -84,7 +84,7 @@ chroot_sessions = True
 
 [vault]
 driver = S3Storage
-url = http://s3:9000
+url = http://vault:9000
 access_key = ${S3_ACCESS_KEY}
 secret_key = ${S3_SECRET_KEY}
 #region = lega
@@ -235,20 +235,20 @@ fi
 
 cat >> ${PRIVATE}/lega.yml <<EOF
   # Stable ID mapper
-  id-mapper:
+  finalize:
     depends_on:
       - db
       - mq
     image: nbisweden/ega-base:dev
-    container_name: id-mapper
+    container_name: finalize
     labels:
-        lega_label: "id-mapper"
+        lega_label: "finalize"
     volumes:
        - ./lega/conf.ini:/etc/ega/conf.ini:ro
     restart: on-failure:3
     networks:
       - lega
-    entrypoint: ["gosu", "lega", "ega-id-mapper"]
+    entrypoint: ["gosu", "lega", "ega-finalize"]
 
   # Ingestion Workers
   ingest:
@@ -358,7 +358,7 @@ cat >> ${PRIVATE}/lega.yml <<EOF
   # Data Out re-encryption service
   res:
     depends_on:
-      - s3
+      - vault
       - keys
     hostname: res
     container_name: res
@@ -379,26 +379,26 @@ cat >> ${PRIVATE}/lega.yml <<EOF
       - EGA_SHAREDPASS_PATH=/etc/ega/pgp/ega.shared.pass
       - EGA_EBI_AWS_ACCESS_KEY=${S3_ACCESS_KEY}
       - EGA_EBI_AWS_ACCESS_SECRET=${S3_SECRET_KEY}
-      - EGA_EBI_AWS_ENDPOINT_URL=http://s3:${DOCKER_PORT_s3}
+      - EGA_EBI_AWS_ENDPOINT_URL=http://vault:${DOCKER_PORT_s3}
       - EGA_EBI_AWS_ENDPOINT_REGION=
     volumes:
       - ./lega/pgp/ega.shared.pass:/etc/ega/pgp/ega.shared.pass:ro
     restart: on-failure:3
     networks:
       - lega
 
-  # S3
-  s3:
-    hostname: s3
-    container_name: s3
+  # Storage backend: S3
+  vault:
+    hostname: vault
+    container_name: vault
     labels:
-        lega_label: "s3"
+        lega_label: "vault"
     image: minio/minio
     environment:
       - MINIO_ACCESS_KEY=${S3_ACCESS_KEY}
       - MINIO_SECRET_KEY=${S3_SECRET_KEY}
     volumes:
-      - s3:/data
+      - vault:/data
     restart: on-failure:3
     networks:
       - lega
@@ -410,7 +410,7 @@ cat >> ${PRIVATE}/lega.yml <<EOF
 volumes:
   db:
   inbox:
-  s3:
+  vault:
 EOF
 
 #########################################################################

docker/bootstrap/settings.rc → deploy/bootstrap/settings.rc

@@ -25,6 +25,14 @@ PGP_PASSPHRASE=$(generate_password 16)
 S3_ACCESS_KEY=$(generate_password 16)
 S3_SECRET_KEY=$(generate_password 32)
 
+# Special cases
+# - Access key length should be between minimum 3 characters in length.
+# - Secret key should be in between 8 and 40 characters.
+if [[ "${DEPLOY_DEV}" = "yes" ]]; then
+    S3_ACCESS_KEY="dummyaccesskey"
+    S3_SECRET_KEY="dummysecretkey"
+fi
+
 LOG_LEVEL=DEBUG
 
 LEGA_PASSWORD=$(generate_password 32)

docker/test/Makefile → deploy/test/Makefile

@@ -19,7 +19,7 @@ get_shasum =                                          \
 
 ##############################
 
-DOCKER_PATH=$(MAIN_REPO)/docker
+DOCKER_PATH=$(MAIN_REPO)/deploy
 CEGA_USERS=$(DOCKER_PATH)/private/cega/users
 
 INSTANCE_PORT=$(shell awk -F= '/DOCKER_PORT_inbox/ {print $$2}' $(DOCKER_PATH)/bootstrap/settings.rc)

docker/tests/pom.xml → deploy/tests/pom.xml

@@ -62,7 +62,7 @@
         <dependency>
             <groupId>no.uio.ifi</groupId>
             <artifactId>crypt4gh</artifactId>
-            <version>1.0.0</version>
+            <version>1.2.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>

docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Uploading.java → deploy/tests/src/test/java/se/nbis/lega/cucumber/steps/Uploading.java

@@ -3,7 +3,7 @@
 import cucumber.api.java8.En;
 import lombok.extern.slf4j.Slf4j;
 import net.schmizz.sshj.sftp.RemoteResourceInfo;
-import no.ifi.uio.crypt4gh.stream.Crypt4GHOutputStream;
+import no.uio.ifi.crypt4gh.stream.Crypt4GHOutputStream;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;

docker/tests/src/test/resources/config.properties → deploy/tests/src/test/resources/config.properties

@@ -11,5 +11,5 @@ container.label.inbox = inbox
 container.label.ingest = ingest
 container.label.keys = keys
 container.label.mq = mq
-container.label.s3 = s3
+container.label.s3 = vault
 container.label.verify = verify

docs/code.rst

@@ -14,7 +14,7 @@ Python Modules
     lega.ingest
     lega.verify
     lega.keyserver
-    lega.mapper
+    lega.finalize
 
 *******************
 Inbox notifications
@@ -31,7 +31,7 @@ Ingestion Worker
    :members:
 .. automodule:: lega.verify
    :members:
-.. automodule:: lega.mapper
+.. automodule:: lega.finalize
    :members:
 
 

setup.py

@@ -26,7 +26,7 @@
               'ega-verify = lega.verify:main',
               'ega-keyserver = lega.keyserver:main',
               'ega-notifications = lega.notifications:main',
-              'ega-id-mapper = lega.mapper:main',
+              'ega-finalize = lega.finalize:main',
               'ega-conf = lega.conf.__main__:main',
           ]
       },

tests/requirements.txt

@@ -4,3 +4,4 @@ aioresponses
 testfixtures
 coveralls
 git+https://github.com/NBISweden/LocalEGA-cryptor.git
+flake8

tests/test_mapper.py → tests/test_finalize.py

@@ -1,12 +1,12 @@
 import unittest
-from lega.mapper import main, work
+from lega.finalize import main, work
 from unittest import mock
 
 
-class testMapper(unittest.TestCase):
-    """Mapper.
+class testFinalize(unittest.TestCase):
+    """Finalize.
 
-    Testing mapper functionalities.
+    Testing Finalizer functionalities.
     """
 
     def setUp(self):
@@ -17,18 +17,18 @@ def tearDown(self):
         """Remove anything that was setup."""
         pass
 
-    @mock.patch('lega.mapper.db')
+    @mock.patch('lega.finalize.db')
     def test_work(self, mock_db):
-        """Test mapper worker, should insert into database."""
+        """Test finalize worker, should insert into database."""
         # mock_db.set_stable_id.return_value = mock.Mock()
         data = {'stable_id': '1', 'file_id': '123'}
         work(data)
         mock_db.set_stable_id.assert_called_with('123', '1')
 
-    @mock.patch('lega.mapper.get_connection')
-    @mock.patch('lega.mapper.consume')
+    @mock.patch('lega.finalize.get_connection')
+    @mock.patch('lega.finalize.consume')
     def test_main(self, mock_consume, mock_connection):
-        """Test main mapper, by mocking cosume call."""
+        """Test main finalize, by mocking cosume call."""
         mock_consume.return_value = mock.MagicMock()
         main()
         mock_consume.assert_called()

tox.ini

@@ -4,8 +4,9 @@ envlist = unit_tests, flake8
 [flake8]
 ignore = E226,D203,D212,D213,D404,D100,D104
 exclude =
-    docker,
+    deploy,
     extras,
+    tmp,
     .tox
 max-line-length = 160
 max-complexity = 10