Fixes #18165: Change the testinfra based tests reports to json format…

… instead of junitxml

rudder-tests/lib/__init__.py

@@ -1,2 +1,2 @@
 from .scenario import ScenarioInterface
-from .reports import XMLReport
+from .reports import XMLReport, JSONReport

rudder-tests/lib/reports.py

@@ -1,4 +1,5 @@
 from lxml import etree
+import json
 
 class XMLReport:
   def __init__(self, path, workspace):
@@ -31,3 +32,61 @@ def merge_reports(self, name, new_report=None):
           print(fin.read())
       except Exception as e:
           print(e)
+
+class JSONReport:
+  def __init__(self, path, workspace):
+    self.path = path
+    self.workspace = workspace
+
+  """
+    Each testinfra test call (== each call to a test != scenario) will produce a report json file.
+    We need to merge them at runtime, and hierachize the report per scenario
+    Resulting json should follow the format:
+    {
+      "scenarios": [
+                     {
+                       "datastate": {},
+                       "summary": {},
+                       "name": "xxxx",
+                       "tests": [
+                                  { "input_data": "",
+                                    "summary": {},
+                                    ...
+                                  },
+                                ]
+                     },
+                   ]
+    }
+  """
+  def merge_reports(self, name, new_report=None, input_data={}, datastate={}):
+    if new_report is None:
+      new_report=self.workspace + "/serverspec-result.xml"
+    try:
+      with open(self.path) as main_json:
+        main_data = json.load(main_json)
+    except:
+        main_data = { "datastate": datastate,
+                      "summary": { "passed": 0, "total": 0, "collected": 0 },
+                      "scenarios": [
+                        { "name": name,
+                          "tests": []
+                        }
+                      ]
+                    }
+    with open(new_report, 'r') as new_json:
+      new_data = json.load(new_json)
+      new_data["input_data"] = input_data
+
+    # Look for the targetted scenario
+    for s in main_data["scenarios"]:
+      if s["name"] == name:
+        # Add current test infos
+        s["tests"].append(new_data)
+
+    # Update the scenario summary
+    main_data["summary"]["total"] += new_data["summary"]["total"]
+    main_data["summary"]["passed"] += new_data["summary"]["passed"]
+    main_data["summary"]["collected"] += new_data["summary"]["collected"]
+
+    with open(self.path, 'w+') as outfile:
+      json.dump(main_data, outfile)

rudder-tests/lib/scenario.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python
+import subprocess
 import tempfile
 import re
 import os
 import json
 import requests
 import traceback
 import pytest
-from lib.reports import XMLReport
+from lib.reports import XMLReport, JSONReport
 from jsonschema import validate, draft7_format_checker, Draft7Validator, RefResolver
 from subprocess import Popen, check_output, PIPE, CalledProcessError
 from time import sleep
@@ -34,7 +35,10 @@ def __init__(self, name, datastate, schema={}):
     self.__set_token()
 
   def __set_token(self):
-    self.token = self.ssh_on(self.nodes("server")[0], "cat /var/rudder/run/api-token")[1]
+    try:
+      self.token = self.ssh_on(self.nodes("server")[0], "cat /var/rudder/run/api-token")[1]
+    except:
+      self.token = ""
 
   # Validate that the given datastate is compatible with the scenario specific
   # required platform
@@ -131,21 +135,30 @@ def ssh_on(self, host, command):
           ssh_cmd = "ssh -i %s %s@%s -p %s %s \"%s\""%(infos["ssh_cred"], infos["ssh_user"], infos["ip"], infos["ssh_port"], options, command)
       return shell(ssh_cmd)
 
-  def push_on(self, host, src, dst):
+  def push_on(self, host, src, dst, recursive=False):
       if host == "localhost":
-          command = "cp %s %s"%(src, dst)
+          if recursive:
+            command = "cp %s %s"%(src, dst)
+          else:
+            command = "cp -r %s %s"%(src, dst)
       else:
           infos = self.datastate[host]
           default_ssh_options = ["StrictHostKeyChecking=no", "UserKnownHostsFile=/dev/null"]
           options = "-o \"" + "\" -o \"".join(default_ssh_options) + "\""
-          command = "scp -i %s -P%s %s  \"%s\" %s@%s:%s"%(infos["ssh_cred"], infos["ssh_port"], options, src, infos["ssh_user"], infos["ip"], dst)
+          if recursive:
+            command = 'scp -r -i %s -P%s %s  "%s" "%s@%s:\\"%s\\""'%(infos["ssh_cred"], infos["ssh_port"], options, src, infos["ssh_user"], infos["ip"], dst)
+          else:
+            # The horrendous syntax is to be fully compatible with windows path, and path using spaces
+            # It should run something like:  scp abc "Administrator@34.240.38.95:\"C:/Program Files/Rudder\""
+            command = 'scp -i %s -P%s %s  "%s" "%s@%s:\\"%s\\""'%(infos["ssh_cred"], infos["ssh_port"], options, src, infos["ssh_user"], infos["ip"], dst)
       return shell(command)
 
   def start(self):
     self.start = datetime.now().isoformat()
     os.makedirs("/tmp/rtf_scenario", exist_ok=True)
     self.workspace = tempfile.mkdtemp(dir="/tmp/rtf_scenario")
-    self.report = XMLReport(self.workspace + "/result.xml", self.workspace)
+    self.report = JSONReport(self.workspace + "/result.xml", self.workspace)
+    #self.report = XMLReport(self.workspace + "/result.xml", self.workspace)
     print(colors.YELLOW + "[" + self.start + "] Begining of scenario " + self.name + colors.RESET)
 
   def finish(self):
@@ -229,11 +242,11 @@ def run(self, target, test, error_mode=Err.CONTINUE, **kwargs):
     All args are passed in a serialized json named test_data
   """
   def run_testinfra(self, target, test, error_mode=Err.CONTINUE, **kwargs):
-    print(colors.BLUE + "Running test %s on %s"%(test, target) + colors.RESET)
-    # prepare command
     input_data = {}
     for k,v in kwargs.items():
       input_data[k.lower()] = v
+    print(colors.BLUE + "Running test %s on %s with:\n%s"%(test, target, json.dumps(input_data)) + colors.RESET)
+    # prepare command
     testfile = "testinfra/tests/" + test + ".py"
     ssh_config_file =  self.workspace + "/ssh_config"
     datastate_to_ssh(target, self.datastate[target], ssh_config_file)
@@ -242,13 +255,15 @@ def run_testinfra(self, target, test, error_mode=Err.CONTINUE, **kwargs):
     except:
       webapp_url = ""
 
-    pytest_cmd = ['-s', '-v', '--test_data', json.dumps(input_data), '--token', self.token, '--webapp_url', webapp_url, testfile, "--junitxml=" + self.report.path]
+    tmp_report_file = self.workspace + "/tmp_report.json"
+    pytest_cmd = ['--capture=no', '-s', '-v', '--test_data=%s'%json.dumps(input_data), '--token=%s'%self.token, '--webapp_url=%s'%webapp_url, testfile, "--json-report", "--json-report-file=" + tmp_report_file]
     if target != "localhost":
-        pytest_cmd = ["--ssh-config=" + ssh_config_file,  "--hosts=" + target] + pytest_cmd
+        pytest_cmd = ["--ssh-config=%s"%ssh_config_file,  "--hosts=%s"%target] + pytest_cmd
+    pytest_cmd = ['pytest'] + pytest_cmd
+    print("+%s"%" ".join(pytest_cmd))
 
-    print("+%s"%pytest_cmd)
-    retcode = pytest.main(pytest_cmd)
-    self.report.merge_reports(self.name)
+    retcode = subprocess.call(pytest_cmd)
+    self.report.merge_reports(self.name, new_report=tmp_report_file, input_data=input_data, datastate=self.datastate)
 
     if retcode != 0:
       if error_mode != Err.IGNORE:

rudder-tests/lib/utils.py

@@ -72,5 +72,8 @@ def datastate_to_ssh(hostname, host, dst):
   User {2}
   Port {3}
   IdentityFile {4}
+  UserKnownHostsFile /dev/null
+  StrictHostKeyChecking no
+  PasswordAuthentication no
 """.format(hostname, host["ip"], host["ssh_user"], host["ssh_port"], host["ssh_cred"]))
 

rudder-tests/scenarios/windows_ncf.py

@@ -21,5 +21,10 @@ def execute(self):
     self.ssh_on("localhost", "git clone --branch " + branch + " git@github.com:Normation/rudder-agent-windows.git " + self.workspace + "/rudder-agent-windows")
     # push it on the agent
     self.push_on(agent, self.workspace + "/rudder-agent-windows/packaging/tests", "C:\Program Files\Rudder", True)
-    self.run_testinfra(agent, "windows_ncf")
+    test_path = "C:/Program Files/Rudder/tests/Command_Execution.Tests.ps1"
+    self.run_testinfra(agent, "windows_ncf", TEST_PATH=test_path)
+
+    test_path = "C:/Program Files/Rudder/tests/Condition_from_variable_match.Tests.ps1"
+    self.run_testinfra(agent, "windows_ncf", TEST_PATH=test_path)
+
     self.finish()

rudder-tests/testinfra/tests/windows_ncf.py

@@ -1,10 +1,15 @@
 import testinfra
+import pytest
+
+# test_path
+@pytest.fixture
+def test_path(test_data):
+    return test_data["test_path"]
 
 """
 Main test
 """
-def test_ncf(host, token, webapp_url):
-  cmd = host.run("powershell.exe /c rudder agent tests")
+def test_ncf(host, token, webapp_url, test_path):
+  cmd = host.run("rudder agent tests -TestFile '%s'"%test_path)
   assert cmd.succeeded
-  print(cmd.stdout)