Merge pull request #28 from EGA-archive/test/cucumber-update

Bring Cucumber test-suite back

.travis.yml

@@ -1,14 +1,16 @@
 language: python
 python: 3.6
 install: true
+cache:
+  directories:
+    - $HOME/.m2
 
 git:
   depth: false
   quiet: true
 
 stages:
   - name: tests
-    if: type IN (push, pull_request)
   - name: images
     if: type = pull_request AND branch = master
 
@@ -71,27 +73,49 @@ jobs:
       script:
         - cd ../tests
         - bats security
-    - stage: images
-      name: "Images Build"
-      script: echo 'Next PR'
-    #
-    # Old tests. Make them succeed with "|| :"
-    #
     - stage: tests
-      name: "Old Tests"
+      name: "Cucumber Ingestion Tests"
+      before_script:
+        - pip3 install -r requirements.txt
+        - cd deploy
+        - make prepare
+        - make bootstrap ARGS='--inbox mina --keyserver ega'
+        - sudo chown -R travis private
+        - make up OMIT="--scale res=0"
+        - make preflight-check
+      script:
+        - cd tests
+        - mvn test -Dtest=IngestionTests
+    - stage: tests
+      name: "Cucumber Outgestion Tests"
       before_script:
         - pip3 install -r requirements.txt
         - cd deploy
         - make prepare
         - make bootstrap ARGS='--inbox mina --keyserver ega'
         - sudo chown -R travis private
         - make up
-        - docker-compose ps
+        - make preflight-check
       script:
         - cd tests
-        - mvn test -Dtest=IngestionTests -B || echo 'Nope'
-        - mvn test -Dtest=OutgestionTests -B || echo 'Nope'
-        - mvn test -Dtest=RobustnessTests -B || echo 'Nope'
+        - mvn test -Dtest=OutgestionTests
+    - stage: tests
+      name: "Cucumber Robustness Tests"
+      before_script:
+        - pip3 install -r requirements.txt
+        - cd deploy
+        - make prepare
+        - make bootstrap ARGS='--inbox mina --keyserver ega'
+        - sudo chown -R travis private
+        - make up OMIT="--scale res=0"
+        - make preflight-check
+      script:
+        - cd tests
+        - mvn test -Dtest=RobustnessTests
+
+    - stage: images
+      name: "Images Build"
+      script: echo 'Next PR'
 
 notifications:
   email: false

deploy/tests/pom.xml

@@ -26,7 +26,7 @@
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <java.version>1.8</java.version>
         <cucumber.version>1.2.5</cucumber.version>
-        <jackson.version>[2.9.8,)</jackson.version>
+        <jackson.version>2.9.8</jackson.version>
         <rabbitmq.version>5.4.0</rabbitmq.version>
     </properties>
 

deploy/tests/src/test/java/se/nbis/lega/cucumber/Context.java

@@ -19,6 +19,7 @@ public class Context {
     private String cegaMQUser;
     private String cegaMQPassword;
     private String cegaMQVHost;
+    private String cegaMQPort;
     private String routingKey;
     private SSHClient ssh;
     private SFTPClient sftp;

deploy/tests/src/test/java/se/nbis/lega/cucumber/Utils.java

@@ -66,7 +66,16 @@ public String getProperty(String key) {
      * @return Absolute path or a private folder.
      */
     public String getPrivateFolderPath() {
-        return Paths.get("").toAbsolutePath().getParent().toString() + getProperty("private.folder.name");
+        return Paths.get("").toAbsolutePath().getParent().getParent().toString() + getProperty("private.folder.path");
+    }
+
+    /**
+     * Gets absolute path or a common folder.
+     *
+     * @return Absolute path or a common folder.
+     */
+    public String getCommonFolderPath() {
+        return Paths.get("").toAbsolutePath().getParent().getParent().toString() + getProperty("common.folder.path");
     }
 
     /**
@@ -119,19 +128,6 @@ public String executeDBQuery(String query) throws IOException, InterruptedExcept
         return executeWithinContainer(dbContainer, "psql", connectionString, "-c", query);
     }
 
-//    /**
-//     * Removes the user's inbox.
-//     *
-//     * @param user     Username.
-//     * @throws InterruptedException In case the query execution is interrupted.
-//     */
-//    public void removeUserInbox(String user) throws InterruptedException {
-//        executeWithinContainer(findContainer(getProperty("images.name.inbox"), getProperty("container.name.inbox")),
-//                String.format("umount -l %s/%s", getProperty("inbox.fuse.folder.path"), user).split(" "));
-//        executeWithinContainer(findContainer(getProperty("images.name.inbox"), getProperty("container.name.inbox")),
-//                String.format("rm -rf %s/%s", getProperty("inbox.real.folder.path"), user).split(" "));
-//    }
-
     /**
      * Removes the uploaded file from the inbox.
      *
@@ -151,7 +147,7 @@ public void removeUploadedFileFromInbox(String user, String fileName) throws Int
      * @throws IOException In case it's not possible to read trace file.
      */
     public String readTraceProperty(String property) throws IOException {
-        File trace = new File(String.format("%s/%s/%s", getPrivateFolderPath(), getProperty("instance.name"), getProperty("trace.file.name")));
+        File trace = new File(String.format("%s/%s", getPrivateFolderPath(), getProperty("trace.file.name")));
         return FileUtils.readLines(trace, Charset.defaultCharset()).
                 stream().
                 filter(l -> l.startsWith(property)).
@@ -199,62 +195,6 @@ public Container stopContainer(Container container) {
         return container;
     }
 
-    /**
-     * Gets all LocalEGA Docker containers.
-     *
-     * @return All LocalEGA Docker containers.
-     */
-    public Collection<Container> getAllLocalEGAContainers() {
-        return dockerClient.listContainersCmd().withShowAll(true).withLabelFilter("lega_label").exec();
-    }
-
-    /**
-     * Restarts all the LocalEGA containers.
-     */
-    public void restartAllLocalEGAContainers() {
-        getAllLocalEGAContainers().
-                stream().
-                peek(this::stopContainer).
-                peek(c -> safeSleep(5000)).
-                peek(this::startContainer).
-                forEach(c -> safeSleep(5000));
-        waitForInitializationToComplete();
-    }
-
-    /**
-     * Waits for all LocalEGA containers to initialize.
-     */
-    public void waitForInitializationToComplete() {
-        Collection<Container> containers = getAllLocalEGAContainers();
-        long maxTimeout = Long.parseLong(getProperty("initialization.max-timeout"));
-        long timeout = 0;
-        while (containers.isEmpty() || !containers.stream().map(Container::getStatus).allMatch(s -> s.startsWith("Up"))) {
-            if (containers.isEmpty()) {
-                containers = getAllLocalEGAContainers();
-            }
-            safeSleep(1000);
-            timeout += 1000;
-            if (timeout > maxTimeout) {
-                throw new RuntimeException(String.format("The system was not initialized in time: initialization.max-timeout = %s", maxTimeout));
-            }
-        }
-        // Sleep a bit more to let containers not only start up, but finish initialization.
-        safeSleep(Long.parseLong(getProperty("initialization.delay")));
-    }
-
-    /**
-     * Sleeps for some time without throwing an exception (to make it easier to use in lambdas).
-     *
-     * @param millis Time to sleep in milliseconds.
-     */
-    private void safeSleep(long millis) {
-        try {
-            Thread.sleep(millis);
-        } catch (InterruptedException e) {
-            log.error(e.getMessage(), e);
-        }
-    }
-
     /**
      * Calculates hash of a file.
      *
@@ -290,20 +230,6 @@ public void publishCEGA(String connection, String user, String encryptedFileName
         message.setFilepath(encryptedFileName);
         message.setStableID("EGAF" + UUID.randomUUID().toString().toLowerCase());
 
-//        if (StringUtils.isNotEmpty(rawChecksum)) {
-//            Checksum unencrypted = new Checksum();
-//            unencrypted.setAlgorithm(hashingAlgorithm.toLowerCase());
-//            unencrypted.setChecksum(rawChecksum);
-//            message.setUnencryptedIntegrity(unencrypted);
-//        }
-//
-//        if (StringUtils.isNotEmpty(encChecksum)) {
-//            Checksum encrypted = new Checksum();
-//            encrypted.setAlgorithm(hashingAlgorithm.toLowerCase());
-//            encrypted.setChecksum(encChecksum);
-//            message.setEncryptedIntegrity(encrypted);
-//        }
-
         ConnectionFactory factory = new ConnectionFactory();
         factory.setUri(connection);
 

deploy/tests/src/test/java/se/nbis/lega/cucumber/hooks/BeforeAfterHooks.java

@@ -26,26 +26,17 @@ public BeforeAfterHooks(Context context) {
     @SuppressWarnings("ResultOfMethodCallIgnored")
     @Before
     public void setUp() throws IOException {
-        context.getUtils().waitForInitializationToComplete();
         File dataFolder = new File("data");
         dataFolder.mkdir();
         File rawFile = File.createTempFile("data", ".raw", dataFolder);
-        FileUtils.writeStringToFile(rawFile, "hello", Charset.defaultCharset());
         context.setDataFolder(dataFolder);
         context.setRawFile(rawFile);
         context.setUser(UUID.randomUUID().toString());
     }
 
-    @SuppressWarnings({"ConstantConditions", "ResultOfMethodCallIgnored"})
     @After
     public void tearDown() throws IOException {
-        Utils utils = context.getUtils();
-
         FileUtils.deleteDirectory(context.getDataFolder());
-        File cegaUsersFolder = new File(utils.getPrivateFolderPath() + "/cega/users/" + utils.getProperty("instance.name"));
-        String user = context.getUser();
-        Arrays.stream(cegaUsersFolder.listFiles((dir, name) -> name.startsWith(user))).forEach(File::delete);
-//        utils.removeUserInbox(user);
     }
 
 }

deploy/tests/src/test/java/se/nbis/lega/cucumber/pojo/FileStatus.java

@@ -18,10 +18,6 @@ public enum FileStatus {
         this.status = status;
     }
 
-    public String getStatus() {
-        return status;
-    }
-
     public static FileStatus getValue(String status) {
         return Arrays.stream(FileStatus.values()).filter(fs -> fs.status.equals(status)).findAny().orElse(FileStatus.UNDEFINED);
     }

deploy/tests/src/test/java/se/nbis/lega/cucumber/pojo/User.java

@@ -0,0 +1,15 @@
+package se.nbis.lega.cucumber.pojo;
+
+import lombok.Data;
+
+@Data
+public class User {
+
+    private String username;
+    private int uid;
+    private String passwordHash;
+    private String gecos;
+    private String sshPublicKey;
+    private Boolean enabled;
+
+}

deploy/tests/src/test/java/se/nbis/lega/cucumber/steps/Authentication.java

@@ -1,5 +1,7 @@
 package se.nbis.lega.cucumber.steps;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
 import cucumber.api.java8.En;
 import lombok.extern.slf4j.Slf4j;
 import net.schmizz.sshj.SSHClient;
@@ -12,37 +14,50 @@
 import org.junit.Assert;
 import se.nbis.lega.cucumber.Context;
 import se.nbis.lega.cucumber.Utils;
+import se.nbis.lega.cucumber.pojo.User;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.security.KeyPair;
 import java.security.KeyPairGenerator;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 import java.util.Arrays;
 import java.util.Base64;
+import java.util.List;
 
 @Slf4j
 public class Authentication implements En {
 
+    private ObjectMapper objectMapper = new ObjectMapper();
+
+    {
+        objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
+    }
+
     public Authentication(Context context) {
         Utils utils = context.getUtils();
 
         Given("^I have an account at Central EGA$", () -> {
-            String cegaUsersFolderPath = utils.getPrivateFolderPath() + "/cega/users/" + utils.getProperty("instance.name");
-            String user = context.getUser();
+            String cegaUsersFolderPath = utils.getCommonFolderPath();
+            String username = context.getUser();
             try {
                 generateKeypair(context);
                 byte[] keyBytes = new Buffer.PlainBuffer().putPublicKey(context.getKeyProvider().getPublic()).getCompactData();
                 String publicKey = Base64.getEncoder().encodeToString(keyBytes);
-                File userYML = new File(String.format(cegaUsersFolderPath + "/%s.yml", user));
-                FileUtils.writeLines(userYML, Arrays.asList("---",
-                        "username: " + user,
-                        "uid: " + Math.abs(new SecureRandom().nextInt()),
-                        "gecos: EGA User " + user,
-                        "pubkey: ssh-rsa " + publicKey));
+                User user = new User();
+                user.setUsername(username);
+                user.setUid(Math.abs(new SecureRandom().nextInt()));
+                user.setSshPublicKey("ssh-rsa " + publicKey);
+                File usersJSON = new File(cegaUsersFolderPath + "/users.json");
+                List<String> lines = FileUtils.readLines(usersJSON, Charset.defaultCharset());
+                lines = lines.subList(0, lines.size() - 1);
+                lines.add("},");
+                lines.add(objectMapper.writeValueAsString(user) + "]");
+                FileUtils.writeLines(usersJSON, lines);
             } catch (IOException e) {
-                log.error(e.getMessage(), e);
+                throw new RuntimeException(e.getMessage(), e);
             }
         });
 
@@ -77,16 +92,6 @@ public Authentication(Context context) {
 
         When("^I am disconnected from the LocalEGA inbox$", () -> Assert.assertFalse(isConnected(context)));
 
-//        When("^inbox is not created for me$", () -> {
-//            try {
-//                disconnect(context);
-//                utils.removeUserInbox(context.getUser());
-//                connect(context);
-//            } catch (InterruptedException e) {
-//                log.error(e.getMessage(), e);
-//            }
-//        });
-
         Then("^I'm logged in successfully$", () -> Assert.assertFalse(context.isAuthenticationFailed()));
 
         Then("^authentication fails$", () -> Assert.assertTrue(context.isAuthenticationFailed()));
@@ -116,7 +121,7 @@ private void connect(Context context) {
         } catch (UserAuthException | SFTPException e) {
             context.setAuthenticationFailed(true);
         } catch (IOException e) {
-            log.error(e.getMessage(), e);
+            throw new RuntimeException(e.getMessage(), e);
         }
     }