Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes #18165: Change the testinfra based tests reports to json format instead of junitxml #252

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion rudder-tests/lib/__init__.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
from .scenario import ScenarioInterface
from .reports import XMLReport
from .reports import XMLReport, JSONReport
59 changes: 59 additions & 0 deletions rudder-tests/lib/reports.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from lxml import etree
import json

class XMLReport:
def __init__(self, path, workspace):
Expand Down Expand Up @@ -31,3 +32,61 @@ def merge_reports(self, name, new_report=None):
print(fin.read())
except Exception as e:
print(e)

class JSONReport:
def __init__(self, path, workspace):
self.path = path
self.workspace = workspace

"""
Each testinfra test call (== each call to a test != scenario) will produce a report json file.
We need to merge them at runtime, and hierachize the report per scenario
Resulting json should follow the format:
{
"scenarios": [
{
"datastate": {},
"summary": {},
"name": "xxxx",
"tests": [
{ "input_data": "",
"summary": {},
...
},
]
},
]
}
"""
def merge_reports(self, name, new_report=None, input_data={}, datastate={}):
if new_report is None:
new_report=self.workspace + "/serverspec-result.xml"
try:
with open(self.path) as main_json:
main_data = json.load(main_json)
except:
main_data = { "datastate": datastate,
"summary": { "passed": 0, "total": 0, "collected": 0 },
"scenarios": [
{ "name": name,
"tests": []
}
]
}
with open(new_report, 'r') as new_json:
new_data = json.load(new_json)
new_data["input_data"] = input_data

# Look for the targetted scenario
for s in main_data["scenarios"]:
if s["name"] == name:
# Add current test infos
s["tests"].append(new_data)

# Update the scenario summary
main_data["summary"]["total"] += new_data["summary"]["total"]
main_data["summary"]["passed"] += new_data["summary"]["passed"]
main_data["summary"]["collected"] += new_data["summary"]["collected"]

with open(self.path, 'w+') as outfile:
json.dump(main_data, outfile)
41 changes: 28 additions & 13 deletions rudder-tests/lib/scenario.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
#!/usr/bin/env python
import subprocess
import tempfile
import re
import os
import json
import requests
import traceback
import pytest
from lib.reports import XMLReport
from lib.reports import XMLReport, JSONReport
from jsonschema import validate, draft7_format_checker, Draft7Validator, RefResolver
from subprocess import Popen, check_output, PIPE, CalledProcessError
from time import sleep
Expand Down Expand Up @@ -34,7 +35,10 @@ def __init__(self, name, datastate, schema={}):
self.__set_token()

def __set_token(self):
self.token = self.ssh_on(self.nodes("server")[0], "cat /var/rudder/run/api-token")[1]
try:
self.token = self.ssh_on(self.nodes("server")[0], "cat /var/rudder/run/api-token")[1]
except:
self.token = ""

# Validate that the given datastate is compatible with the scenario specific
# required platform
Expand Down Expand Up @@ -131,21 +135,30 @@ def ssh_on(self, host, command):
ssh_cmd = "ssh -i %s %s@%s -p %s %s \"%s\""%(infos["ssh_cred"], infos["ssh_user"], infos["ip"], infos["ssh_port"], options, command)
return shell(ssh_cmd)

def push_on(self, host, src, dst):
def push_on(self, host, src, dst, recursive=False):
if host == "localhost":
command = "cp %s %s"%(src, dst)
if recursive:
command = "cp %s %s"%(src, dst)
else:
command = "cp -r %s %s"%(src, dst)
else:
infos = self.datastate[host]
default_ssh_options = ["StrictHostKeyChecking=no", "UserKnownHostsFile=/dev/null"]
options = "-o \"" + "\" -o \"".join(default_ssh_options) + "\""
command = "scp -i %s -P%s %s \"%s\" %s@%s:%s"%(infos["ssh_cred"], infos["ssh_port"], options, src, infos["ssh_user"], infos["ip"], dst)
if recursive:
command = 'scp -r -i %s -P%s %s "%s" "%s@%s:\\"%s\\""'%(infos["ssh_cred"], infos["ssh_port"], options, src, infos["ssh_user"], infos["ip"], dst)
else:
# The horrendous syntax is to be fully compatible with windows path, and path using spaces
# It should run something like: scp abc "Administrator@34.240.38.95:\"C:/Program Files/Rudder\""
command = 'scp -i %s -P%s %s "%s" "%s@%s:\\"%s\\""'%(infos["ssh_cred"], infos["ssh_port"], options, src, infos["ssh_user"], infos["ip"], dst)
return shell(command)

def start(self):
self.start = datetime.now().isoformat()
os.makedirs("/tmp/rtf_scenario", exist_ok=True)
self.workspace = tempfile.mkdtemp(dir="/tmp/rtf_scenario")
self.report = XMLReport(self.workspace + "/result.xml", self.workspace)
self.report = JSONReport(self.workspace + "/result.xml", self.workspace)
#self.report = XMLReport(self.workspace + "/result.xml", self.workspace)
print(colors.YELLOW + "[" + self.start + "] Begining of scenario " + self.name + colors.RESET)

def finish(self):
Expand Down Expand Up @@ -229,11 +242,11 @@ def run(self, target, test, error_mode=Err.CONTINUE, **kwargs):
All args are passed in a serialized json named test_data
"""
def run_testinfra(self, target, test, error_mode=Err.CONTINUE, **kwargs):
print(colors.BLUE + "Running test %s on %s"%(test, target) + colors.RESET)
# prepare command
input_data = {}
for k,v in kwargs.items():
input_data[k.lower()] = v
print(colors.BLUE + "Running test %s on %s with:\n%s"%(test, target, json.dumps(input_data)) + colors.RESET)
# prepare command
testfile = "testinfra/tests/" + test + ".py"
ssh_config_file = self.workspace + "/ssh_config"
datastate_to_ssh(target, self.datastate[target], ssh_config_file)
Expand All @@ -242,13 +255,15 @@ def run_testinfra(self, target, test, error_mode=Err.CONTINUE, **kwargs):
except:
webapp_url = ""

pytest_cmd = ['-s', '-v', '--test_data', json.dumps(input_data), '--token', self.token, '--webapp_url', webapp_url, testfile, "--junitxml=" + self.report.path]
tmp_report_file = self.workspace + "/tmp_report.json"
pytest_cmd = ['--capture=no', '-s', '-v', '--test_data=%s'%json.dumps(input_data), '--token=%s'%self.token, '--webapp_url=%s'%webapp_url, testfile, "--json-report", "--json-report-file=" + tmp_report_file]
if target != "localhost":
pytest_cmd = ["--ssh-config=" + ssh_config_file, "--hosts=" + target] + pytest_cmd
pytest_cmd = ["--ssh-config=%s"%ssh_config_file, "--hosts=%s"%target] + pytest_cmd
pytest_cmd = ['pytest'] + pytest_cmd
print("+%s"%" ".join(pytest_cmd))

print("+%s"%pytest_cmd)
retcode = pytest.main(pytest_cmd)
self.report.merge_reports(self.name)
retcode = subprocess.call(pytest_cmd)
self.report.merge_reports(self.name, new_report=tmp_report_file, input_data=input_data, datastate=self.datastate)

if retcode != 0:
if error_mode != Err.IGNORE:
Expand Down
3 changes: 3 additions & 0 deletions rudder-tests/lib/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,5 +72,8 @@ def datastate_to_ssh(hostname, host, dst):
User {2}
Port {3}
IdentityFile {4}
UserKnownHostsFile /dev/null
StrictHostKeyChecking no
PasswordAuthentication no
""".format(hostname, host["ip"], host["ssh_user"], host["ssh_port"], host["ssh_cred"]))

7 changes: 6 additions & 1 deletion rudder-tests/scenarios/windows_ncf.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,10 @@ def execute(self):
self.ssh_on("localhost", "git clone --branch " + branch + " git@github.com:Normation/rudder-agent-windows.git " + self.workspace + "/rudder-agent-windows")
# push it on the agent
self.push_on(agent, self.workspace + "/rudder-agent-windows/packaging/tests", "C:\Program Files\Rudder", True)
self.run_testinfra(agent, "windows_ncf")
test_path = "C:/Program Files/Rudder/tests/Command_Execution.Tests.ps1"
self.run_testinfra(agent, "windows_ncf", TEST_PATH=test_path)

test_path = "C:/Program Files/Rudder/tests/Condition_from_variable_match.Tests.ps1"
self.run_testinfra(agent, "windows_ncf", TEST_PATH=test_path)

self.finish()
11 changes: 8 additions & 3 deletions rudder-tests/testinfra/tests/windows_ncf.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
import testinfra
import pytest

# test_path
@pytest.fixture
def test_path(test_data):
return test_data["test_path"]

"""
Main test
"""
def test_ncf(host, token, webapp_url):
cmd = host.run("powershell.exe /c rudder agent tests")
def test_ncf(host, token, webapp_url, test_path):
cmd = host.run("rudder agent tests -TestFile '%s'"%test_path)
assert cmd.succeeded
print(cmd.stdout)