[CI] Upload test results to CircleCI for LTE tests and LTE-IntegTests (…

…#4974)

* Upload LteTest results to CircleCI

Signed-off-by: Marie Bremner <marwhal@fb.com>

* Collect test outputs from s1ap tests

Signed-off-by: Marie Bremner <marwhal@fb.com>

* Collect test outputs from s1ap tests

Signed-off-by: Marie Bremner <marwhal@fb.com>

.circleci/config.yml

@@ -733,6 +733,7 @@ jobs:
       - checkout
       - build/determinator:
           <<: *lte_build_verify
+      - run: mkdir -p /var/tmp/test_results
       - run: sudo apt-get update -y
       - run: sudo apt-get install -y libsystemd-dev pkg-config curl zip unzip
       - run: sudo apt-get install -y virtualenv python-babel python-dev build-essential python3-setuptools python-setuptools autogen autoconf libtool python3-apt python3-aioeventlet python3-requests python3-pip python-protobuf
@@ -748,6 +749,8 @@ jobs:
             mkdir ${CODEGEN_ROOT}
             wget https://repo1.maven.org/maven2/io/swagger/swagger-codegen-cli/2.2.3/swagger-codegen-cli-2.2.3.jar -O $SWAGGER_CODEGEN_JAR
             make -C $MAGMA_ROOT/lte/gateway/python test_all
+      - store_test_results:
+          path: /var/tmp/test_results
       - magma_slack_notify
 
   lte-integ-test:

circleci/fabfile.py

@@ -209,13 +209,23 @@ def _run_remote_lte_integ_test(repo: str, magma_root: str):
     repo_name = _get_repo_name(repo)
     with cd(f'{repo_name}/{magma_root}/lte/gateway'):
         test_result = run('fab integ_test', timeout=120*60, warn_only=True)
+
+        # Transfer test summaries into current directory
+        run('fab get_test_summaries:dst_path="test-results"', warn_only=True)
+        # Copy from node
+        local('mkdir -p test-results')
+        get('test-results', 'test-results')
+        # Copy to the directory CircleCI expects
+        local('sudo mkdir -p /tmp/test-results/')
+        local('sudo mv test-results/* /tmp/test-results/')
+
         # On failure, transfer logs from all 3 VMs and copy to the log
         # directory. This will get stored as an artifact in the CircleCI
         # config.
         if test_result.return_code:
             tar_file_name = "lte-test-logs.tar.gz"
             # On failure, transfer logs into current directory
-            log_path = './lte-test-logs.tar.gz'
+            log_path = './' + tar_file_name
             run(f'fab get_test_logs:dst_path="{log_path}"', warn_only=True)
             # Copy the log files out from the node
             local('mkdir lte-artifacts')

lte/gateway/deploy/roles/dev_common/tasks/main.yml

@@ -308,3 +308,8 @@
   become: yes
   command: "update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-7 1000"
   when: preburn
+
+- file:
+    path: /var/tmp/test_results
+    state: directory
+    mode: 0777

lte/gateway/fabfile.py

@@ -51,6 +51,7 @@
 AGW_INTEG_ROOT = "$MAGMA_ROOT/lte/gateway/python/integ_tests"
 DEFAULT_CERT = "$MAGMA_ROOT/.cache/test_certs/rootCA.pem"
 DEFAULT_PROXY = "$MAGMA_ROOT/lte/gateway/configs/control_proxy.yml"
+TEST_SUMMARY_GLOB = "/var/tmp/test_results/*.xml"
 
 # Look for keys as specified in our ~/.ssh/config
 env.use_ssh_config = True
@@ -260,6 +261,18 @@ def run_integ_tests():
     gateway_ip = '192.168.60.142'
     execute(_run_integ_tests, gateway_ip)
 
+def get_test_summaries(
+        gateway_host=None,
+        test_host=None,
+        dst_path="/tmp"):
+    local('mkdir -p ' + dst_path)
+    _switch_to_vm(gateway_host, "magma", "magma_dev.yml", False)
+    with settings(warn_only=True):
+        get(remote_path=TEST_SUMMARY_GLOB, local_path=dst_path)
+    _switch_to_vm(test_host, "magma_test", "magma_test.yml", False)
+    with settings(warn_only=True):
+        get(remote_path=TEST_SUMMARY_GLOB, local_path=dst_path)
+
 
 def get_test_logs(gateway_host=None,
                   test_host=None,
@@ -453,3 +466,10 @@ def _run_integ_tests(gateway_ip='192.168.60.142'):
           ' export GATEWAY_IP=%s;'
           ' make integ_test\''
           % (key, host, port, gateway_ip))
+
+
+def _switch_to_vm(addr, host_name, ansible_file, destroy_vm):
+    if not addr:
+        vagrant_setup(host_name, destroy_vm)
+    else:
+        ansible_setup(addr, host_name, ansible_file)

lte/gateway/python/integ_tests/Makefile

@@ -21,9 +21,10 @@ $(PYTHON_BUILD)/setupinteg_env:
 	touch $(PYTHON_BUILD)/setupinteg_env
 
 # TODO T21489739 - Don't sleep and don't stop after a failure
+RESULTS_DIR := /var/tmp/test_results
 define execute_test
  	echo "Running test: $(1)"
-	sudo -E PATH=$(PATH) PYTHONPATH=$(PYTHONPATH):$(S1AP_TESTER_PYTHON_PATH) $(PYTHON_BUILD)/bin/nosetests -x -s $(1) || exit 1
+	sudo -E PATH=$(PATH) PYTHONPATH=$(PYTHONPATH):$(S1AP_TESTER_PYTHON_PATH) $(PYTHON_BUILD)/bin/nosetests --with-xunit --xunit-file=$(RESULTS_DIR)/$(basename $(notdir $(1))).xml -x -s $(1) || exit 1
 	sleep 1
 endef
 

orc8r/gateway/python/python.mk

@@ -107,15 +107,19 @@ $(BIN)/grpcio-tools: install_virtualenv
 
 .test: .tests .sudo_tests
 
+RESULTS_DIR := /var/tmp/test_results
+
 .tests:
 ifdef TESTS
-	. $(PYTHON_BUILD)/bin/activate; $(BIN)/nosetests --with-coverage --cover-erase --cover-branches --cover-package=magma -s $(TESTS)
+	$(eval NAME ?= $(shell $(BIN)/python setup.py --name))
+	. $(PYTHON_BUILD)/bin/activate; $(BIN)/nosetests --with-xunit --xunit-file=$(RESULTS_DIR)/tests_$(NAME).xml --with-coverage --cover-erase --cover-branches --cover-package=magma -s $(TESTS)
 endif
 
 .sudo_tests:
 ifdef SUDO_TESTS
 ifndef SKIP_SUDO_TESTS
-	. $(PYTHON_BUILD)/bin/activate; sudo $(BIN)/nosetests --with-coverage --cover-branches --cover-package=magma -s $(SUDO_TESTS)
+	$(eval NAME ?= $(shell $(BIN)/python setup.py --name))
+	. $(PYTHON_BUILD)/bin/activate; sudo $(BIN)/nosetests --with-xunit --xunit-file=$(RESULTS_DIR)/sudo_$(NAME).xml --with-coverage --cover-branches --cover-package=magma -s $(SUDO_TESTS)
 endif
 endif