EFA Test Fixes

Signed-off-by: Sean Smith <seaam@amazon.com>

tests/integration-tests/cfn_stacks_factory.py

@@ -20,7 +20,7 @@
 class CfnStack:
     """Identify a CloudFormation stack."""
 
-    def __init__(self, name, region, template=None):
+    def __init__(self, name, region, template):
         self.name = name
         self.region = region
         self.template = template

tests/integration-tests/clusters_factory.py

@@ -21,7 +21,7 @@
 class Cluster:
     """Contain all static and dynamic data related to a cluster instance."""
 
-    def __init__(self, name, ssh_key, config_file=None):
+    def __init__(self, name, ssh_key, config_file):
         self.name = name
         self.config_file = config_file
         self.ssh_key = ssh_key
@@ -121,7 +121,7 @@ def create_cluster(self, cluster):
         # create the cluster
         logging.info("Creating cluster {0} with config {1}".format(name, config))
         self.__created_clusters[name] = cluster
-        result = run_command(["pcluster", "create", "--no-rollback", "--config", config, name])
+        result = run_command(["pcluster", "create", "--norollback", "--config", config, name])
         if "Status: {0} - CREATE_COMPLETE".format(cluster.cfn_name) not in result.stdout:
             error = "Cluster creation failed for {0} with output: {1}".format(name, result.stdout)
             logging.error(error)

tests/integration-tests/conftest.py

@@ -166,15 +166,15 @@ def clusters_factory(request):
     factory = ClustersFactory()
 
     def _cluster_factory(cluster_config):
-        if request.config.getoption("cluster"):
-            cluster = Cluster(name=request.config.getoption("cluster"), ssh_key=request.config.getoption("key_path"))
-        else:
-            cluster_config = _write_cluster_config_to_outdir(request, cluster_config)
-            cluster = Cluster(
-                name="integ-tests-" + random_alphanumeric(),
-                config_file=cluster_config,
-                ssh_key=request.config.getoption("key_path"),
-            )
+        cluster_config = _write_cluster_config_to_outdir(request, cluster_config)
+        cluster = Cluster(
+            name=request.config.getoption("cluster")
+            if request.config.getoption("cluster")
+            else "integ-tests-" + random_alphanumeric(),
+            config_file=cluster_config,
+            ssh_key=request.config.getoption("key_path"),
+        )
+        if not request.config.getoption("cluster"):
             factory.create_cluster(cluster)
         return cluster
 
@@ -338,7 +338,8 @@ def vpc_stacks(cfn_stacks_factory, request):
 @retry(stop_max_attempt_number=2, wait_fixed=5000)
 def _create_vpc_stack(request, template, region, cfn_stacks_factory):
     if request.config.getoption("vpc_stack"):
-        stack = CfnStack(name=request.config.getoption("vpc_stack"), region=region)
+        logging.info("Using stack {0} in region {1}".format(request.config.getoption("vpc_stack"), region))
+        stack = CfnStack(name=request.config.getoption("vpc_stack"), region=region, template=template.to_json())
     else:
         stack = CfnStack(name="integ-tests-vpc-" + random_alphanumeric(), region=region, template=template.to_json())
         cfn_stacks_factory.create_stack(stack)

tests/integration-tests/tests/test_efa/test_efa.py

@@ -10,6 +10,7 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 import logging
+import re
 
 import pytest
 
@@ -20,7 +21,7 @@
 INSTANCES_TO_SLOTS_MAP = {"c5n.18xlarge": 72, "p3dn.24xlarge": 96, "i3en.24xlarge": 96}
 
 
-@pytest.mark.regions(["us-east-1", "eu-west-1"])
+@pytest.mark.regions(["us-east-1"])
 @pytest.mark.instances(["c5n.18xlarge", "p3dn.24xlarge", "i3en.24xlarge"])
 @pytest.mark.oss(["alinux", "centos7", "ubuntu1604"])
 @pytest.mark.schedulers(["sge", "slurm"])
@@ -47,7 +48,7 @@ def _test_efa_installed(scheduler_commands, remote_command_executor):
     # Output contains:
     # 00:06.0 Ethernet controller: Amazon.com, Inc. Device efa0
     logging.info("Testing EFA installed")
-    result = scheduler_commands.submit_command("/sbin/lspci > /shared/lspci.out")
+    result = scheduler_commands.submit_command("lspci > /shared/lspci.out")
 
     job_id = scheduler_commands.assert_job_submitted(result.stdout)
     scheduler_commands.wait_job_completed(job_id)
@@ -58,7 +59,7 @@ def _test_efa_installed(scheduler_commands, remote_command_executor):
     assert_that(result.stdout).contains("00:06.0 Ethernet controller: Amazon.com, Inc. Device efa0")
 
     # Check EFA interface not present on master
-    result = remote_command_executor.run_remote_command("/sbin/lspci")
+    result = remote_command_executor.run_remote_command("lspci")
     assert_that(result.stdout).does_not_contain("00:06.0 Ethernet controller: Amazon.com, Inc. Device efa0")
 
 
@@ -91,5 +92,6 @@ def _test_osu_benchmarks(remote_command_executor, scheduler_commands, test_datad
     scheduler_commands.wait_job_completed(job_id)
     scheduler_commands.assert_job_succeeded(job_id)
 
-    # TODO: perform assertions on benchmarks results
-    remote_command_executor.run_remote_command("cat /shared/osu.out")
+    output = remote_command_executor.run_remote_command("cat /shared/osu.out").stdout
+    latency = re.search(r"0\s+(\d\d)\.", output).group(1)
+    assert_that(int(latency)).is_less_than(20)

tests/integration-tests/tests/test_efa/test_efa/test_efa/init_osu_benchmarks.sh

@@ -6,5 +6,4 @@ wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.4
 tar zxvf ./osu-micro-benchmarks-5.4.tar.gz
 cd osu-micro-benchmarks-5.4/
 ./configure CC=/opt/amazon/efa/bin/mpicc CXX=/opt/amazon/efa/bin/mpicxx
-make
-# make install in the submit script
+make

tests/integration-tests/tests/test_efa/test_efa/test_efa/mpi_submit.sh

@@ -2,4 +2,4 @@
 set -e
 
 module load openmpi
-mpirun -N 1 -np 2 "mpi_hello_world" &> /shared/mpi.out
+mpirun -N 1 -np 2 "mpi_hello_world" >> /shared/mpi.out

tests/integration-tests/tests/test_efa/test_efa/test_efa/osu_submit.sh

@@ -2,4 +2,4 @@
 set -e
 
 module load openmpi
-mpirun --map-by ppr:1:node /shared/osu-micro-benchmarks-5.4/mpi/pt2pt/osu_latency &> /shared/osu.out
+mpirun --map-by ppr:1:node /shared/osu-micro-benchmarks-5.4/mpi/pt2pt/osu_latency >> /shared/osu.out