Merge pull request #3295 from input-output-hk/improve-vm-tests

Improve NixOS vm test asserts
IntersectMBO · Jun 7, 2021 · cc938ad · cc938ad
2 parents 74cb849 + 4245aba
commit cc938ad
Show file tree

Hide file tree

Showing 5 changed files with 81 additions and 36 deletions.
diff --git a/nix/tests/vm-tests/all.nix b/nix/tests/vm-tests/all.nix
@@ -66,7 +66,7 @@ makeTest {
       ];
 
       networking = {
-        firewall.allowedTCPPorts = [ 8080 9090 ];
+        firewall.allowedTCPPorts = [ 7070 8080 9090 ];
         extraHosts = ''
           127.0.0.1 plutus-playground
           127.0.0.1 marlowe-playground
@@ -209,6 +209,15 @@ makeTest {
     #
     playgrounds.wait_for_unit("network-online.target")
     pab.wait_for_unit("network-online.target")
+    pab.wait_for_unit("pab.service")
+
+    # Refer to `services.pab` configuration  above to see what
+    # service each individual port relates to.
+    pab.wait_for_open_port(8080)
+    pab.wait_for_open_port(8081)
+    pab.wait_for_open_port(8082)
+    pab.wait_for_open_port(8083)
+    pab.wait_for_open_port(8085)
     webghc.wait_for_unit("network-online.target")
     playgrounds.succeed("ping -c1 192.168.1.1")
     playgrounds.succeed("ping -c1 192.168.1.2")
@@ -230,12 +239,31 @@ makeTest {
     playgrounds.wait_for_open_port(7070)
     playgrounds.wait_for_open_port(8080)
     playgrounds.wait_for_open_port(9090)
-    playgrounds.succeed("curl --silent http://plutus-playground:8080/ | grep  'plutus'")
-    playgrounds.succeed("curl --silent http://plutus-playground:8080/doc/ | grep 'The Plutus Platform'")
-    playgrounds.succeed("curl --silent http://plutus-playground:8080/doc/plutus/tutorials/ | grep 'Tutorials'")
-    playgrounds.succeed("curl --silent http://marlowe-playground:9090/ | grep 'marlowe-playground'")
-    playgrounds.succeed("curl --silent http://marlowe-playground:9090/doc/ | grep 'The Plutus Platform'")
-    playgrounds.succeed("curl --silent http://marlowe-playground:9090/doc/marlowe/tutorials/ | grep 'Tutorials'")
+
+    with subtest("********************************************************************************************* TEST: All content is being served on playgrounds"):
+      res = playgrounds.succeed("curl --silent http://plutus-playground:8080/")
+      assert "plutus" in res, "Expected string 'plutus' from 'http://plutus-playground:8080'. Actual: {}".format(res)
+
+      res = playgrounds.succeed("curl --silent http://plutus-playground:8080/doc/")
+      assert "The Plutus Platform" in res, "Expected string 'The Plutus Platform' from 'http://plutus-playground:8080/doc/'. Actual: {}".format(res)
+
+      res = playgrounds.succeed("curl --silent http://plutus-playground:8080/doc/plutus/tutorials/")
+      assert "The Plutus Platform" in res, "Expected string 'Tutorials' from 'http://plutus-playground:8080/doc/plutus/tutorials/'. Actual: {}".format(res)
+
+      res = playgrounds.succeed("curl --silent http://marlowe-playground:9090/")
+      assert "marlowe-playground" in res, "Expected string 'marlowe-playground' from 'http://marlowe-playground:9090'. Actual: {}".format(res)
+
+      res = playgrounds.succeed("curl --silent http://marlowe-playground:9090/doc/")
+      assert "marlowe-playground" in res, "Expected string 'The Plutus Platform' from 'http://marlowe-playground:9090/doc'. Actual: {}".format(res)
+
+      res = playgrounds.succeed("curl --silent http://marlowe-playground:9090/doc/marlowe/tutorials/")
+      assert "Tutorials" in res, "Expected string 'Tutorials' from 'http://marlowe-playground:9090/doc/marlowe/tutorials'. Actual: {}".format(res)
+
+      #res = playgrounds.succeed("curl --silent http://marlowe-dashboard:7070/")
+      #assert "marlowe-dashboard" in res, "Expected string 'marlowe-dashboard' from 'http://marlowe-dashboard:7070/'. Actual: {}".format(res)
+
+      res = pab.succeed("curl --silent http://localhost:8080/")
+      assert "marlowe-dashboard" in res, "Expected string 'marlowe-dashboard' from 'http://marlowe-dashboard:7070/'. Actual: {}".format(res)
 
     #
     # webghc asserts
@@ -246,20 +274,17 @@ makeTest {
     #
     # pab asserts
     #
-    pab.wait_for_unit("pab.service")
 
-    #
-    # marlowe-dashboard asserts
-    #
-    playgrounds.succeed("curl --silent http://marlowe-dashboard:7070/ | grep 'marlowe-dashboard'")
   '' + lib.optionalString (vmCompileTests) ''
     #
     # plutus-playground / webghc : using api/contract
     # marlowe-playground / webghc : using /runghc
     #
-    playgrounds.succeed("curl --silent http://webghc/health")
-    playgrounds.succeed("curl --silent -H 'Content-Type: application/json' --request POST --data @${plutusApiRequest} http://plutus-playground:8080/api/contract | grep Right ")
-    playgrounds.succeed("curl --silent http://webghc/health")
-    playgrounds.succeed("curl --silent -H 'Content-Type: application/json' --request POST --data @${marloweApiRequest} http://marlowe-playground:9090/runghc | grep Right")
+    with subtest("********************************************************************************************* TEST: compilation works"):
+      res = playgrounds.succeed("curl --silent -H 'Content-Type: application/json' --request POST --data @${plutusApiRequest} http://plutus-playground:8080/api/contract")
+      assert "Right" in res, "Expected response wrapped in 'Right'. Actual: {}".format(res)
+
+      res = playgrounds.succeed("curl --silent -H 'Content-Type: application/json' --request POST --data @${marloweApiRequest} http://marlowe-playground:9090/runghc")
+      assert "Right" in res, "Expected response wrapped in 'Right'. Actual: {}".format(res)
   '';
 }
diff --git a/nix/tests/vm-tests/marlowe-playground.nix b/nix/tests/vm-tests/marlowe-playground.nix
@@ -5,11 +5,12 @@ let
     FRONTEND_URL="http://localhost:8080"
     GITHUB_CALLBACK_PATH="/#/gh-oauth-cb"
     GITHUB_CLIENT_ID="314123123a312fe"
-    GITHUB_CLIENT_SECRET=kljfks234dskjhfeskjr"
+    GITHUB_CLIENT_SECRET="kljfks234dskjhfeskjr"
   '';
 in
 makeTest {
   name = "marlowe";
+  skipLint = true;
   machine = { pkgs, ... }:
     {
       imports = [ ../../modules/marlowe-playground.nix ];
@@ -29,13 +30,15 @@ makeTest {
     machine.wait_for_unit("marlowe-playground.service")
     machine.wait_for_open_port(4001)
     machine.succeed("curl localhost:4001/health")
-    machine.succeed("mkdir -p /var/lib/playgrounds")
-    machine.succeed("cp ${envFile} /var/lib/playgrounds/marlowe.env")
-    machine.succeed("systemctl restart marlowe-playground")
-    machine.wait_for_unit("marlowe-playground.service")
-    # TODO: verify it is using the file - for some reason
-    # i am having problems verifying this with a 'journalctl |grep'
-    # even though i can see the output in the terminal.
+
+    with subtest("********************************************************************************************* TEST: Can reload a config file"):
+      machine.succeed("mkdir -p /var/lib/playgrounds")
+      machine.succeed("cp ${envFile} /var/lib/playgrounds/marlowe.env")
+      machine.succeed("systemctl restart marlowe-playground")
+      machine.wait_for_unit("marlowe-playground.service")
+
+      res = machine.succeed("journalctl -u marlowe-playground.service --no-pager")
+      assert "Loading environment config from '/var/lib/playgrounds/marlowe.env'" in res, "Expected playground to load config. Actual: {}".format(res)
   '';
 
 }
diff --git a/nix/tests/vm-tests/pab.nix b/nix/tests/vm-tests/pab.nix
@@ -1,6 +1,7 @@
 { makeTest, plutus-pab, marlowe-dashboard }:
 makeTest {
   name = "pab";
+  skipLint = true;
   machine = { pkgs, ... }:
     {
       imports = [ ../../modules/pab.nix ];
@@ -28,8 +29,15 @@ makeTest {
     machine.wait_for_open_port(8083)
     machine.wait_for_open_port(8081)
     machine.wait_for_open_port(8082)
-    machine.succeed("curl -s localhost:8080 | grep marlowe-dashboard")
-    machine.succeed("pab-exec contracts installed | grep '/tmp/file-that-does-not-exist'")
+
+    with subtest("********************************************************************************************* TEST: Serves static files from config"):
+      res = machine.succeed("curl -s localhost:8080 | grep marlowe-dashboard")
+      assert "marlowe-dashboard" in res, "Expected string 'marlowe-dashboard' in served content. Actual: {}".format(res)
+
+    with subtest("********************************************************************************************* TEST: Loads contracts in config"):
+      res = machine.succeed("pab-exec contracts installed")
+      assert "/tmp/file-that-does-not-exist" in res, "Expected '/tmp/file-that-does-not-exist' in output. Actual: {}".format(res)
+
   '';
 
 }
diff --git a/nix/tests/vm-tests/plutus-playground.nix b/nix/tests/vm-tests/plutus-playground.nix
@@ -10,6 +10,7 @@ let
 in
 makeTest {
   name = "plutus-playground";
+  skipLint = true;
   machine = { pkgs, ... }:
     {
       imports = [ ../../modules/plutus-playground.nix ];
@@ -29,14 +30,15 @@ makeTest {
     machine.succeed("systemctl start plutus-playground")
     machine.wait_for_unit("plutus-playground.service")
     machine.wait_for_open_port(4000)
-    machine.succeed("curl localhost:4000/api/version")
-    machine.succeed("mkdir -p /var/lib/playgrounds")
-    machine.succeed("cp ${envFile} /var/lib/playgrounds/plutus.env")
-    machine.succeed("systemctl restart plutus-playground")
-    machine.wait_for_unit("plutus-playground.service")
-    # TODO: verify it is using the file - for some reason
-    # i am having problems verifying this with a 'journalctl |grep'
-    # even though i can see the output in the terminal.
+
+    with subtest("********************************************************************************************* TEST: Can reload a config file"):
+      machine.succeed("mkdir -p /var/lib/playgrounds")
+      machine.succeed("cp ${envFile} /var/lib/playgrounds/plutus.env")
+      machine.succeed("systemctl restart plutus-playground")
+      machine.wait_for_unit("plutus-playground.service")
+
+      res = machine.succeed("journalctl -u plutus-playground.service --no-pager")
+      assert "Loading environment config from '/var/lib/playgrounds/plutus.env'" in res, "Expected playground to load config. Actual: {}".format(res)
   '';
 
 }
diff --git a/nix/tests/vm-tests/web-ghc.nix b/nix/tests/vm-tests/web-ghc.nix
@@ -17,6 +17,7 @@ let
 in
 makeTest {
   name = "web-ghc";
+  skipLint = true;
   machine = { pkgs, ... }:
     {
       imports = [ ../../modules/web-ghc.nix ];
@@ -32,8 +33,14 @@ makeTest {
     machine.start()
     machine.wait_for_unit("web-ghc.service")
     machine.wait_for_open_port(80)
-    machine.succeed("curl -sSfL -H 'Content-Type: application/json' --request POST --data @${validCompileRequest} http://localhost:80/runghc 2>&1 | grep Right")
-    machine.succeed("curl -sSfL -H 'Content-Type: application/json' --request POST --data @${invalidCompileRequest} http://localhost:80/runghc 2>&1 | grep Left")
+
+    with subtest("********************************************************************************************* TEST: Can process a valid compilation request"):
+      response = machine.succeed("curl -sSfL -H 'Content-Type: application/json' --request POST --data @${validCompileRequest} http://localhost:80/runghc")
+      assert "Right" in response, "Expected response wrapped in 'Right'. Actual: {}".format(response)
+
+    with subtest("********************************************************************************************* TEST: Can process a invalid compilation request"):
+      response = machine.succeed("curl -sSfL -H 'Content-Type: application/json' --request POST --data @${invalidCompileRequest} http://localhost:80/runghc")
+      assert "Left" in response, "Expected response wrapped in 'Left'. Actual: {}".format(response)
   '';
 
 }