Add RES microservice to the Docker Compose setup

.travis.yml

@@ -53,9 +53,20 @@ jobs:
         - make up
         - docker-compose ps
       script:
-        - sleep 25
         - cd tests
-        - mvn test -Dtest=CommonTests -B
+        - mvn test -Dtest=IngestionTests -B
+    - stage: integration tests
+      before_script:
+        - cd docker
+        - ../extras/travis_push_docker_hub.sh
+        - make bootstrap ARGS='--inbox mina --keyserver ega'
+        - sudo chown -R travis private
+        - docker network create cega
+        - make up
+        - docker-compose ps
+      script:
+        - cd tests
+        - mvn test -Dtest=OutgestionTests -B
     - stage: integration tests
       before_script:
         - cd docker
@@ -66,7 +77,6 @@ jobs:
         - make up
         - docker-compose ps
       script:
-        - sleep 25
         - cd tests
         - mvn test -Dtest=RobustnessTests -B
     - stage: integration tests

docker/bootstrap/cega.sh

@@ -162,6 +162,8 @@ services:
       - "5672:5672"
     image: rabbitmq:3.6.14-management
     container_name: cega-mq
+    labels:
+        lega_label: "cega-mq"
     volumes:
        - ./cega/mq/defs.json:/etc/rabbitmq/defs.json:ro
        - ./cega/mq/rabbitmq.config:/etc/rabbitmq/rabbitmq.config:ro
@@ -174,6 +176,8 @@ services:
     image: nbisweden/ega-base:dev
     hostname: cega-users
     container_name: cega-users
+    labels:
+        lega_label: "cega-users"
     #ports:
     #  - "9100:80"
     expose:
@@ -187,7 +191,9 @@ services:
     networks:
       - cega
     command: ["python3.6", "/cega/server.py"]
-
+EOF
+if [[ $KEYSERVER != 'ega' ]]; then
+cat >> ${PRIVATE}/cega.yml <<EOF
   ############################################
   # Fake Eureka server
   ############################################
@@ -199,13 +205,16 @@ services:
       - 8761
     image: nbisweden/ega-base:dev
     container_name: cega-eureka
+    labels:
+        lega_label: "cega-eureka"
     volumes:
       - ../images/cega/eureka.py:/cega/eureka.py
     restart: on-failure:3
     networks:
       - cega
     command: ["python3.6", "/cega/eureka.py"]
 EOF
+fi
 
 # Only one instance, called 'lega'
 cat > ${PRIVATE}/cega/env <<EOF

docker/bootstrap/lega.sh

@@ -144,6 +144,8 @@ services:
       - "${DOCKER_PORT_mq}:15672"
     image: rabbitmq:3.6.14-management
     container_name: mq
+    labels:
+        lega_label: "mq"
     restart: on-failure:3
     # Required external link
     external_links:
@@ -168,6 +170,8 @@ services:
       - PGDATA=/ega/data
     hostname: db
     container_name: db
+    labels:
+        lega_label: "db"
     image: postgres:9.6
     volumes:
       - db:/ega/data
@@ -185,6 +189,8 @@ services:
     external_links:
       - cega-users:cega-users
     container_name: inbox
+    labels:
+        lega_label: "inbox"
     restart: on-failure:3
     networks:
       - lega
@@ -224,6 +230,8 @@ cat >> ${PRIVATE}/lega.yml <<EOF
       - mq
     image: nbisweden/ega-base:dev
     container_name: id-mapper
+    labels:
+        lega_label: "id-mapper"
     volumes:
        - ./lega/conf.ini:/etc/ega/conf.ini:ro
     restart: on-failure:3
@@ -238,6 +246,8 @@ cat >> ${PRIVATE}/lega.yml <<EOF
       - mq
     image: nbisweden/ega-base:dev
     container_name: ingest
+    labels:
+        lega_label: "ingest"
     environment:
       - S3_ACCESS_KEY=${S3_ACCESS_KEY}
       - S3_SECRET_KEY=${S3_SECRET_KEY}
@@ -258,7 +268,9 @@ cat >> ${PRIVATE}/lega.yml <<EOF
   keys:
     hostname: keys
     container_name: keys
-    image: nbisweden/keys
+    labels:
+        lega_label: "keys"
+    image: cscfi/ega-keyserver
     environment:
       - SPRING_PROFILES_ACTIVE=no-oss
       - EGA_KEY_PATH=/etc/ega/pgp/ega.sec,/etc/ega/pgp/ega2.sec
@@ -282,6 +294,8 @@ cat >> ${PRIVATE}/lega.yml <<EOF
   keys:
     hostname: keys
     container_name: keys
+    labels:
+        lega_label: "keys"
     image: nbisweden/ega-base:dev
     expose:
       - "8443"
@@ -314,6 +328,8 @@ cat >> ${PRIVATE}/lega.yml <<EOF
       - keys
     hostname: verify
     container_name: verify
+    labels:
+        lega_label: "verify"
     image: nbisweden/ega-base:dev
     environment:
       - LEGA_PASSWORD=${LEGA_PASSWORD}
@@ -328,10 +344,44 @@ cat >> ${PRIVATE}/lega.yml <<EOF
       - lega
     entrypoint: ["gosu", "lega", "ega-verify"]
 
+  # Data Out re-encryption service
+  res:
+    depends_on:
+      - s3
+      - keys
+    hostname: res
+    container_name: res
+    labels:
+        lega_label: "res"
+    image: cscfi/ega-res
+    ports:
+      - "${DOCKER_PORT_res}:8080"
+    environment:
+      - SPRING_PROFILES_ACTIVE=no-oss,LocalEGA
+      - EGA_EGA_EXTERNAL_URL=
+      - EGA_EGA_CRAM_FASTA_A=
+      - EGA_EGA_CRAM_FASTA_B=
+      - EGA_EBI_FIRE_URL=
+      - EGA_EBI_FIRE_ARCHIVE=
+      - EGA_EBI_FIRE_KEY=
+      - SERVICE_ARCHIVE_CLASS=
+      - EGA_SHAREDPASS_PATH=/etc/ega/pgp/ega.shared.pass
+      - EGA_EBI_AWS_ACCESS_KEY=${S3_ACCESS_KEY}
+      - EGA_EBI_AWS_ACCESS_SECRET=${S3_SECRET_KEY}
+      - EGA_EBI_AWS_ENDPOINT_URL=http://s3:${DOCKER_PORT_s3}
+      - EGA_EBI_AWS_ENDPOINT_REGION=
+    volumes:
+      - ./lega/pgp/ega.shared.pass:/etc/ega/pgp/ega.shared.pass:ro
+    restart: on-failure:3
+    networks:
+      - lega
+
   # S3
   s3:
     hostname: s3
     container_name: s3
+    labels:
+        lega_label: "s3"
     image: minio/minio
     environment:
       - MINIO_ACCESS_KEY=${S3_ACCESS_KEY}
@@ -382,7 +432,7 @@ S3_SECRET_KEY             = ${S3_SECRET_KEY}
 DOCKER_PORT_inbox         = ${DOCKER_PORT_inbox}
 DOCKER_PORT_mq            = ${DOCKER_PORT_mq}
 DOCKER_PORT_s3            = ${DOCKER_PORT_s3}
-DOCKER_PORT_kibana        = ${DOCKER_PORT_kibana}
+DOCKER_PORT_res           = ${DOCKER_PORT_res}
 #
 LEGA_PASSWORD             = ${LEGA_PASSWORD}
 KEYS_PASSWORD             = ${KEYS_PASSWORD}

docker/bootstrap/settings.rc

@@ -6,6 +6,7 @@ DOCKER_PORT_mq=15672
 DOCKER_PORT_kibana=5601
 DOCKER_PORT_s3=9000
 DOCKER_PORT_keyserver=8443
+DOCKER_PORT_res=8081
 
 CEGA_MQ_PASSWORD=$(generate_password 16)
 [[ "${DEPLOY_DEV}" = "yes" ]] && CEGA_MQ_PASSWORD=lega # overwrite

docker/tests/src/test/java/se/nbis/lega/cucumber/Context.java

@@ -25,6 +25,9 @@ public class Context {
     private File dataFolder;
     private File rawFile;
     private File encryptedFile;
+    private File downloadedFile;
+    private String sessionKey;
+    private String iv;
     private Map<String, String> ingestionInformation;
 
     private boolean authenticationFailed;

docker/tests/src/test/java/se/nbis/lega/cucumber/CommonTests.java → docker/tests/src/test/java/se/nbis/lega/cucumber/IngestionTests.java

@@ -13,5 +13,5 @@
                 "src/test/resources/cucumber/features/ingestion.feature"
         }
 )
-public class CommonTests {
+public class IngestionTests {
 }

docker/tests/src/test/java/se/nbis/lega/cucumber/OutgestionTests.java

@@ -0,0 +1,15 @@
+package se.nbis.lega.cucumber;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+        format = {"pretty", "html:target/cucumber"},
+        features = {
+                "src/test/resources/cucumber/features/outgestion.feature"
+        }
+)
+public class OutgestionTests {
+}

docker/tests/src/test/java/se/nbis/lega/cucumber/Utils.java

@@ -11,19 +11,22 @@
 import com.rabbitmq.client.Connection;
 import com.rabbitmq.client.ConnectionFactory;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
+import org.c02e.jpgpj.HashingAlgorithm;
 import se.nbis.lega.cucumber.publisher.Message;
 
 import javax.ws.rs.core.MediaType;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Paths;
-import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Properties;
 import java.util.UUID;
 
@@ -107,7 +110,7 @@ public String executeWithinContainer(Container container, String... command) thr
      * @throws InterruptedException In case the query execution is interrupted.
      */
     public String executeDBQuery(String query) throws IOException, InterruptedException {
-        return executeWithinContainer(findContainer(getProperty("images.name.db"), getProperty("container.name.db")),
+        return executeWithinContainer(findContainer(getProperty("container.label.db")),
                 "psql", "-U", readTraceProperty("DB_USER"), "-d", "lega", "-c", query);
     }
 
@@ -131,7 +134,7 @@ public String executeDBQuery(String query) throws IOException, InterruptedExcept
      * @throws InterruptedException In case the query execution is interrupted.
      */
     public void removeUploadedFileFromInbox(String user, String fileName) throws InterruptedException {
-        executeWithinContainer(findContainer(getProperty("images.name.inbox"), getProperty("container.name.inbox")),
+        executeWithinContainer(findContainer(getProperty("container.label.inbox")),
                 String.format("rm %s/%s/%s", getProperty("inbox.folder.path"), user, fileName).split(" "));
     }
 
@@ -156,17 +159,17 @@ public String readTraceProperty(String property) throws IOException {
     /**
      * Finds container by image name and container name.
      *
-     * @param imageName     Image name.
-     * @param containerName Container name.
+     * @param label Container lavbel.
      * @return Docker container.
      */
-    public Container findContainer(String imageName, String containerName) {
-        return dockerClient.listContainersCmd().withShowAll(true).exec().
-                stream().
-                filter(c -> c.getImage().equals(imageName)).
-                filter(c -> ArrayUtils.contains(c.getNames(), "/" + containerName)).
-                findAny().
-                orElse(null);
+    public Container findContainer(String label) {
+        return dockerClient
+                .listContainersCmd()
+                .withShowAll(true)
+                .withLabelFilter(Collections.singletonMap("lega_label", label)).exec()
+                .stream()
+                .findAny()
+                .orElseThrow(() -> new RuntimeException(String.format("Container with label %s not found!", label)));
     }
 
     /**
@@ -191,14 +194,21 @@ public Container stopContainer(Container container) {
         return container;
     }
 
+    /**
+     * Gets all LocalEGA Docker containers.
+     *
+     * @return All LocalEGA Docker containers.
+     */
+    public Collection<Container> getAllLocalEGAContainers() {
+        return dockerClient.listContainersCmd().withShowAll(true).withLabelFilter("lega_label").exec();
+    }
+
     /**
      * Restarts all the LocalEGA containers (the ones that starts with `ega-` or `ega-` prefix).
      */
     public void restartAllLocalEGAContainers() {
-        dockerClient.listContainersCmd().withShowAll(true).exec().
+        getAllLocalEGAContainers().
                 stream().
-                filter(c -> Arrays.stream(c.getNames()).anyMatch(n -> n.startsWith("/" + getProperty("container.name") + "-")
-                        || n.startsWith("/" + getProperty("container.name") + "_"))).
                 peek(this::stopContainer).
                 peek(c -> safeSleep(5000)).
                 peek(this::startContainer).
@@ -219,26 +229,26 @@ private void safeSleep(long millis) {
         }
     }
 
-//    /**
-//     * Calculates hash of a file.
-//     *
-//     * @param file             File to calculate hash for.
-//     * @param hashingAlgorithm Algorithm to use for hashing.
-//     * @return Hash. Defaults to MD5.
-//     * @throws IOException In case it's not possible ot read the file.
-//     */
-//    public String calculateChecksum(File file, HashingAlgorithm hashingAlgorithm) throws IOException {
-//        try (FileInputStream fileInputStream = new FileInputStream(file)) {
-//            switch (hashingAlgorithm) {
-//                case SHA256:
-//                    return DigestUtils.sha256Hex(fileInputStream);
-//                case MD5:
-//                    return DigestUtils.md5Hex(fileInputStream);
-//                default:
-//                    throw new RuntimeException(hashingAlgorithm + " hashing algorithm is not supported by the test-suite.");
-//            }
-//        }
-//    }
+    /**
+     * Calculates hash of a file.
+     *
+     * @param file             File to calculate hash for.
+     * @param hashingAlgorithm Algorithm to use for hashing.
+     * @return Hash. Defaults to MD5.
+     * @throws IOException In case it's not possible ot read the file.
+     */
+    public String calculateChecksum(File file, HashingAlgorithm hashingAlgorithm) throws IOException {
+        try (FileInputStream fileInputStream = new FileInputStream(file)) {
+            switch (hashingAlgorithm) {
+                case SHA256:
+                    return DigestUtils.sha256Hex(fileInputStream);
+                case MD5:
+                    return DigestUtils.md5Hex(fileInputStream);
+                default:
+                    throw new RuntimeException(hashingAlgorithm + " hashing algorithm is not supported by the test-suite.");
+            }
+        }
+    }
 
     /**
      * Sends a JSON message to a RabbitMQ instance.