diff --git a/.travis.yml b/.travis.yml index 800c6e9c..a317f86b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,18 @@ jobs: - make -C images base inbox - docker push nbisweden/ega-base:"PR${TRAVIS_PULL_REQUEST}" - docker push nbisweden/ega-inbox:"PR${TRAVIS_PULL_REQUEST}" + - stage: integration tests + before_script: + - cd docker + - ../extras/travis_push_docker_hub.sh + - make bootstrap ARGS='--inbox mina --keyserver ega' + - sudo chown -R travis private + - docker network create cega + - make up-omit service=res + - docker-compose ps + script: + - cd tests + - mvn test -Dtest=IngestionTests -B - stage: integration tests before_script: - cd docker @@ -53,9 +65,8 @@ jobs: - make up - docker-compose ps script: - - sleep 25 - cd tests - - mvn test -Dtest=CommonTests -B + - mvn test -Dtest=OutgestionTests -B - stage: integration tests before_script: - cd docker @@ -66,7 +77,6 @@ jobs: - make up - docker-compose ps script: - - sleep 25 - cd tests - mvn test -Dtest=RobustnessTests -B - stage: integration tests diff --git a/docker/Makefile b/docker/Makefile index 44ca1ac1..2fa05ad3 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -24,6 +24,10 @@ network: up:network private/cega.yml private/lega.yml @docker-compose -f private/cega.yml -f private/lega.yml up -d +# Omiting just one service from the setup, to avoid bootstrap flags +up-omit:network private/cega.yml private/lega.yml + @docker-compose -f private/cega.yml -f private/lega.yml up -d --scale ${service}=0 + clean-volumes: docker volume rm lega_db lega_inbox lega_s3 diff --git a/docker/bootstrap/cega.sh b/docker/bootstrap/cega.sh index cb41f02c..6d187b7b 100644 --- a/docker/bootstrap/cega.sh +++ b/docker/bootstrap/cega.sh @@ -162,6 +162,8 @@ services: - "5672:5672" image: rabbitmq:3.6.14-management container_name: cega-mq + labels: + lega_label: "cega-mq" volumes: - ./cega/mq/defs.json:/etc/rabbitmq/defs.json:ro - ./cega/mq/rabbitmq.config:/etc/rabbitmq/rabbitmq.config:ro @@ -174,6 +176,8 @@ services: image: nbisweden/ega-base:dev hostname: cega-users container_name: cega-users + labels: + lega_label: "cega-users" #ports: # - "9100:80" expose: @@ -187,7 +191,9 @@ services: networks: - cega command: ["python3.6", "/cega/server.py"] - +EOF +if [[ $KEYSERVER != 'ega' ]]; then +cat >> ${PRIVATE}/cega.yml < ${PRIVATE}/cega/env <> ${PRIVATE}/lega.yml <> ${PRIVATE}/lega.yml <> ${PRIVATE}/lega.yml <> ${PRIVATE}/lega.yml <> ${PRIVATE}/lega.yml <> ${PRIVATE}/lega.yml < ingestionInformation; private boolean authenticationFailed; diff --git a/docker/tests/src/test/java/se/nbis/lega/cucumber/CommonTests.java b/docker/tests/src/test/java/se/nbis/lega/cucumber/IngestionTests.java similarity index 94% rename from docker/tests/src/test/java/se/nbis/lega/cucumber/CommonTests.java rename to docker/tests/src/test/java/se/nbis/lega/cucumber/IngestionTests.java index 27b0b43d..df56e569 100644 --- a/docker/tests/src/test/java/se/nbis/lega/cucumber/CommonTests.java +++ b/docker/tests/src/test/java/se/nbis/lega/cucumber/IngestionTests.java @@ -13,5 +13,5 @@ "src/test/resources/cucumber/features/ingestion.feature" } ) -public class CommonTests { +public class IngestionTests { } diff --git a/docker/tests/src/test/java/se/nbis/lega/cucumber/OutgestionTests.java b/docker/tests/src/test/java/se/nbis/lega/cucumber/OutgestionTests.java new file mode 100644 index 00000000..77690673 --- /dev/null +++ b/docker/tests/src/test/java/se/nbis/lega/cucumber/OutgestionTests.java @@ -0,0 +1,15 @@ +package se.nbis.lega.cucumber; + +import cucumber.api.CucumberOptions; +import cucumber.api.junit.Cucumber; +import org.junit.runner.RunWith; + +@RunWith(Cucumber.class) +@CucumberOptions( + format = {"pretty", "html:target/cucumber"}, + features = { + "src/test/resources/cucumber/features/outgestion.feature" + } +) +public class OutgestionTests { +} diff --git a/docker/tests/src/test/java/se/nbis/lega/cucumber/Utils.java b/docker/tests/src/test/java/se/nbis/lega/cucumber/Utils.java index 1443c08e..502dc106 100644 --- a/docker/tests/src/test/java/se/nbis/lega/cucumber/Utils.java +++ b/docker/tests/src/test/java/se/nbis/lega/cucumber/Utils.java @@ -11,19 +11,22 @@ import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; +import org.c02e.jpgpj.HashingAlgorithm; import se.nbis.lega.cucumber.publisher.Message; import javax.ws.rs.core.MediaType; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; -import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.Properties; import java.util.UUID; @@ -107,7 +110,7 @@ public String executeWithinContainer(Container container, String... command) thr * @throws InterruptedException In case the query execution is interrupted. */ public String executeDBQuery(String query) throws IOException, InterruptedException { - return executeWithinContainer(findContainer(getProperty("images.name.db"), getProperty("container.name.db")), + return executeWithinContainer(findContainer(getProperty("container.label.db")), "psql", "-U", readTraceProperty("DB_USER"), "-d", "lega", "-c", query); } @@ -131,7 +134,7 @@ public String executeDBQuery(String query) throws IOException, InterruptedExcept * @throws InterruptedException In case the query execution is interrupted. */ public void removeUploadedFileFromInbox(String user, String fileName) throws InterruptedException { - executeWithinContainer(findContainer(getProperty("images.name.inbox"), getProperty("container.name.inbox")), + executeWithinContainer(findContainer(getProperty("container.label.inbox")), String.format("rm %s/%s/%s", getProperty("inbox.folder.path"), user, fileName).split(" ")); } @@ -156,17 +159,17 @@ public String readTraceProperty(String property) throws IOException { /** * Finds container by image name and container name. * - * @param imageName Image name. - * @param containerName Container name. + * @param label Container lavbel. * @return Docker container. */ - public Container findContainer(String imageName, String containerName) { - return dockerClient.listContainersCmd().withShowAll(true).exec(). - stream(). - filter(c -> c.getImage().equals(imageName)). - filter(c -> ArrayUtils.contains(c.getNames(), "/" + containerName)). - findAny(). - orElse(null); + public Container findContainer(String label) { + return dockerClient + .listContainersCmd() + .withShowAll(true) + .withLabelFilter(Collections.singletonMap("lega_label", label)).exec() + .stream() + .findAny() + .orElseThrow(() -> new RuntimeException(String.format("Container with label %s not found!", label))); } /** @@ -191,14 +194,21 @@ public Container stopContainer(Container container) { return container; } + /** + * Gets all LocalEGA Docker containers. + * + * @return All LocalEGA Docker containers. + */ + public Collection getAllLocalEGAContainers() { + return dockerClient.listContainersCmd().withShowAll(true).withLabelFilter("lega_label").exec(); + } + /** * Restarts all the LocalEGA containers (the ones that starts with `ega-` or `ega-` prefix). */ public void restartAllLocalEGAContainers() { - dockerClient.listContainersCmd().withShowAll(true).exec(). + getAllLocalEGAContainers(). stream(). - filter(c -> Arrays.stream(c.getNames()).anyMatch(n -> n.startsWith("/" + getProperty("container.name") + "-") - || n.startsWith("/" + getProperty("container.name") + "_"))). peek(this::stopContainer). peek(c -> safeSleep(5000)). peek(this::startContainer). @@ -219,26 +229,26 @@ private void safeSleep(long millis) { } } -// /** -// * Calculates hash of a file. -// * -// * @param file File to calculate hash for. -// * @param hashingAlgorithm Algorithm to use for hashing. -// * @return Hash. Defaults to MD5. -// * @throws IOException In case it's not possible ot read the file. -// */ -// public String calculateChecksum(File file, HashingAlgorithm hashingAlgorithm) throws IOException { -// try (FileInputStream fileInputStream = new FileInputStream(file)) { -// switch (hashingAlgorithm) { -// case SHA256: -// return DigestUtils.sha256Hex(fileInputStream); -// case MD5: -// return DigestUtils.md5Hex(fileInputStream); -// default: -// throw new RuntimeException(hashingAlgorithm + " hashing algorithm is not supported by the test-suite."); -// } -// } -// } + /** + * Calculates hash of a file. + * + * @param file File to calculate hash for. + * @param hashingAlgorithm Algorithm to use for hashing. + * @return Hash. Defaults to MD5. + * @throws IOException In case it's not possible ot read the file. + */ + public String calculateChecksum(File file, HashingAlgorithm hashingAlgorithm) throws IOException { + try (FileInputStream fileInputStream = new FileInputStream(file)) { + switch (hashingAlgorithm) { + case SHA256: + return DigestUtils.sha256Hex(fileInputStream); + case MD5: + return DigestUtils.md5Hex(fileInputStream); + default: + throw new RuntimeException(hashingAlgorithm + " hashing algorithm is not supported by the test-suite."); + } + } + } /** * Sends a JSON message to a RabbitMQ instance. diff --git a/docker/tests/src/test/java/se/nbis/lega/cucumber/hooks/BeforeAfterHooks.java b/docker/tests/src/test/java/se/nbis/lega/cucumber/hooks/BeforeAfterHooks.java index 57db8ceb..69d41567 100644 --- a/docker/tests/src/test/java/se/nbis/lega/cucumber/hooks/BeforeAfterHooks.java +++ b/docker/tests/src/test/java/se/nbis/lega/cucumber/hooks/BeforeAfterHooks.java @@ -1,5 +1,6 @@ package se.nbis.lega.cucumber.hooks; +import com.github.dockerjava.api.model.Container; import cucumber.api.java.After; import cucumber.api.java.Before; import cucumber.api.java8.En; @@ -12,7 +13,9 @@ import java.io.IOException; import java.nio.charset.Charset; import java.util.Arrays; +import java.util.Collection; import java.util.UUID; +import java.util.concurrent.TimeoutException; @Slf4j public class BeforeAfterHooks implements En { @@ -25,7 +28,8 @@ public BeforeAfterHooks(Context context) { @SuppressWarnings("ResultOfMethodCallIgnored") @Before - public void setUp() throws IOException { + public void setUp() throws IOException, TimeoutException, InterruptedException { + waitForInitializationToComplete(context.getUtils()); File dataFolder = new File("data"); dataFolder.mkdir(); File rawFile = File.createTempFile("data", ".raw", dataFolder); @@ -35,6 +39,24 @@ public void setUp() throws IOException { context.setUser(UUID.randomUUID().toString()); } + public void waitForInitializationToComplete(Utils utils) throws InterruptedException, TimeoutException { + Collection containers = utils.getAllLocalEGAContainers(); + long maxTimeout = Long.parseLong(utils.getProperty("initialization.max-timeout")); + long timeout = 0; + while (containers.isEmpty() || !containers.stream().map(Container::getStatus).allMatch(s -> s.startsWith("Up"))) { + if (containers.isEmpty()) { + containers = utils.getAllLocalEGAContainers(); + } + Thread.sleep(1000); + timeout += 1000; + if (timeout > maxTimeout) { + throw new TimeoutException(String.format("The system was not initialized in time: initialization.max-timeout = %s", maxTimeout)); + } + } + // Sleep a bit more to let containers not only start up, but finish initialization. + Thread.sleep(Long.parseLong(utils.getProperty("initialization.delay"))); + } + @SuppressWarnings({"ConstantConditions", "ResultOfMethodCallIgnored"}) @After public void tearDown() throws IOException { diff --git a/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Authentication.java b/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Authentication.java index 72c1601c..18b28674 100644 --- a/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Authentication.java +++ b/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Authentication.java @@ -36,7 +36,11 @@ public Authentication(Context context) { byte[] keyBytes = new Buffer.PlainBuffer().putPublicKey(context.getKeyProvider().getPublic()).getCompactData(); String publicKey = Base64.getEncoder().encodeToString(keyBytes); File userYML = new File(String.format(cegaUsersFolderPath + "/%s.yml", user)); - FileUtils.writeLines(userYML, Arrays.asList("---", "pubkey: ssh-rsa " + publicKey)); + FileUtils.writeLines(userYML, Arrays.asList("---", + "username: " + user, + "uid: " + Math.abs(new SecureRandom().nextInt()), + "gecos: EGA User " + user, + "pubkey: ssh-rsa " + publicKey)); } catch (IOException e) { log.error(e.getMessage(), e); } diff --git a/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Ingestion.java b/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Ingestion.java index 09fa1ffd..1c705a5d 100644 --- a/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Ingestion.java +++ b/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Ingestion.java @@ -38,23 +38,17 @@ public Ingestion(Context context) { } }); - When("^I turn off the keyserver$", () -> utils.stopContainer(utils.findContainer(utils.getProperty("images.name.keys"), - utils.getProperty("container.name.keys")))); + When("^I turn off the keyserver$", () -> utils.stopContainer(utils.findContainer(utils.getProperty("container.label.keys")))); - When("^I turn on the keyserver$", () -> utils.startContainer(utils.findContainer(utils.getProperty("images.name.keys"), - utils.getProperty("container.name.keys")))); + When("^I turn on the keyserver$", () -> utils.startContainer(utils.findContainer(utils.getProperty("container.label.keys")))); - When("^I turn off the database", () -> utils.stopContainer(utils.findContainer(utils.getProperty("images.name.db"), - utils.getProperty("container.name.db")))); + When("^I turn off the database", () -> utils.stopContainer(utils.findContainer(utils.getProperty("container.label.db")))); - When("^I turn on the database", () -> utils.startContainer(utils.findContainer(utils.getProperty("images.name.db"), - utils.getProperty("container.name.db")))); + When("^I turn on the database", () -> utils.startContainer(utils.findContainer(utils.getProperty("container.label.db")))); - When("^I turn off the message broker", () -> utils.stopContainer(utils.findContainer(utils.getProperty("images.name.mq"), - utils.getProperty("container.name.mq")))); + When("^I turn off the message broker", () -> utils.stopContainer(utils.findContainer(utils.getProperty("container.label.mq")))); - When("^I turn on the message broker", () -> utils.startContainer(utils.findContainer(utils.getProperty("images.name.mq"), - utils.getProperty("container.name.mq")))); + When("^I turn on the message broker", () -> utils.startContainer(utils.findContainer(utils.getProperty("container.label.mq")))); When("^I ingest file from the LocalEGA inbox$", () -> { // HashingAlgorithm hashingAlgorithm = HashingAlgorithm.valueOf(algorithm); diff --git a/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Outgetsion.java b/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Outgetsion.java new file mode 100644 index 00000000..545c7e82 --- /dev/null +++ b/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Outgetsion.java @@ -0,0 +1,51 @@ +package se.nbis.lega.cucumber.steps; + +import cucumber.api.java8.En; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.c02e.jpgpj.HashingAlgorithm; +import org.junit.Assert; +import se.nbis.lega.cucumber.Context; +import se.nbis.lega.cucumber.Utils; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.Map; + +@Slf4j +public class Outgetsion implements En { + + public Outgetsion(Context context) { + Utils utils = context.getUtils(); + + When("^I download archived file$", () -> { + try { + Map ingestionInformation = context.getIngestionInformation(); + String filePath = ingestionInformation.get("vault_path"); + URL resURL = new URL(String.format("http://localhost:8081/file?sourceKey=%s&sourceIV=%s&filePath=%s", + context.getSessionKey(), + context.getIv(), + filePath)); + File downloadedFile = new File(context.getRawFile().getAbsolutePath() + ".out"); + context.setDownloadedFile(downloadedFile); + FileUtils.copyURLToFile(resURL, downloadedFile); + } catch (IOException e) { + log.error(e.getMessage(), e); + Assert.fail(e.getMessage()); + } + }); + + Then("^checksums of raw and downloaded files match$", () -> { + try { + String rawChecksum = utils.calculateChecksum(context.getRawFile(), HashingAlgorithm.SHA256); + String downloadedChecksum = utils.calculateChecksum(context.getDownloadedFile(), HashingAlgorithm.SHA256); + Assert.assertEquals(rawChecksum, downloadedChecksum); + } catch (IOException e) { + log.error(e.getMessage(), e); + Assert.fail(e.getMessage()); + } + }); + } + +} diff --git a/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Uploading.java b/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Uploading.java index e53b04b2..b2e1a068 100644 --- a/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Uploading.java +++ b/docker/tests/src/test/java/se/nbis/lega/cucumber/steps/Uploading.java @@ -4,6 +4,7 @@ import lombok.extern.slf4j.Slf4j; import net.schmizz.sshj.sftp.RemoteResourceInfo; import no.ifi.uio.crypt4gh.stream.Crypt4GHOutputStream; +import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.junit.Assert; @@ -37,6 +38,8 @@ public Uploading(Context context) { String key = FileUtils.readFileToString(new File(String.format("%s/%s/pgp/ega.pub", utils.getPrivateFolderPath(), utils.getProperty("instance.name"))), Charset.defaultCharset()); FileOutputStream fileOutputStream = new FileOutputStream(encryptedFile); Crypt4GHOutputStream crypt4GHOutputStream = new Crypt4GHOutputStream(fileOutputStream, key, digest); + context.setSessionKey(Hex.encodeHexString(crypt4GHOutputStream.getSessionKeyBytes())); + context.setIv(Hex.encodeHexString(crypt4GHOutputStream.getIvBytes())); FileUtils.copyFile(rawFile, crypt4GHOutputStream); crypt4GHOutputStream.close(); context.setEncryptedFile(encryptedFile); diff --git a/docker/tests/src/test/resources/config.properties b/docker/tests/src/test/resources/config.properties index 295f733a..0b4b7d36 100644 --- a/docker/tests/src/test/resources/config.properties +++ b/docker/tests/src/test/resources/config.properties @@ -2,20 +2,14 @@ private.folder.name = /private instance.name = lega trace.file.name = .trace inbox.folder.path = /ega/inbox +initialization.max-timeout = 100000 +initialization.delay = 100000 ingest.max-timeout = 100000 -images.name.db = postgres:9.6 -images.name.inbox = nbisweden/ega-mina-inbox -images.name.ingest = nbisweden/ega-base:dev -images.name.keys = nbisweden/keys -images.name.mq = rabbitmq:3.6.14-management -images.name.s3 = minio/minio -images.name.verify = nbisweden/ega-base:dev - -container.name.db = db -container.name.inbox = inbox -container.name.ingest = ingest -container.name.keys = keys -container.name.mq = mq -container.name.s3 = s3 -container.name.verify = verify +container.label.db = db +container.label.inbox = inbox +container.label.ingest = ingest +container.label.keys = keys +container.label.mq = mq +container.label.s3 = s3 +container.label.verify = verify diff --git a/docker/tests/src/test/resources/cucumber/features/outgestion.feature b/docker/tests/src/test/resources/cucumber/features/outgestion.feature new file mode 100644 index 00000000..b2373516 --- /dev/null +++ b/docker/tests/src/test/resources/cucumber/features/outgestion.feature @@ -0,0 +1,14 @@ +Feature: Outgestion + As a user I want to be able to retrieve files from the LocalEGA + + Scenario: O.0 User retrieves the file, previously ingested with Crypt4GH format + Given I have an account at Central EGA + And I have correct private key + And I connect to the LocalEGA inbox via SFTP using private key + And I have a 1 MB file encrypted with Crypt4GH using a LocalEGA's pubic key + And I upload encrypted file to the LocalEGA inbox via SFTP + And I have CEGA MQ username and password + And I ingest file from the LocalEGA inbox + And I retrieve ingestion information + When I download archived file + Then checksums of raw and downloaded files match