Support a junit_xml reporter

examples/server_client/configs/master.py

@@ -24,5 +24,5 @@
 properties = {
   "additional_paths": [
     ROOT_DIR
-  ]
+  ],
 }

requirements.txt

@@ -4,3 +4,4 @@ pytz>=2013.8
 jinja2
 luminol
 python-dateutil
+junit_xml

setup.py

@@ -65,7 +65,8 @@ def list_files(directory):
       'pytz>=2012c',
       'jinja2>=2.7.3',
       'python-dateutil',
-      'kazoo>=1.1'
+      'kazoo>=1.1',
+      'junit_xml>=1.6'
   ],
   entry_points = {
       'console_scripts': [

test/samples/sample_junit_configs/master.py

@@ -0,0 +1,26 @@
+# Copyright 2014 LinkedIn Corp.
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+ROOT_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+
+properties = {
+  "junit_reporter":True
+}

test/samples/sample_junit_configs/sample_config1.txt

@@ -0,0 +1,2 @@
+TEST_OPTION=foo
+BAR=baz

test/samples/sample_test_junit_reports.py

@@ -0,0 +1,118 @@
+# Copyright 2014 LinkedIn Corp.
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import zopkio.remote_host_helper as remote_host_helper
+import shutil
+from time import sleep
+__test__ = False  # don't this as a test
+
+TEST_DIRECTORY = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+LOGS_DIRECTORY = "/tmp/test_with_naarad/collected_logs/"
+OUTPUT_DIRECTORY = "/tmp/test_with_naarad/results/"
+__ssh__ = None
+__sample_code_in__ = None
+
+test = {
+  "deployment_code": os.path.abspath(__file__),
+  "test_code": [os.path.abspath(__file__)],
+  "dynamic_configuration_code": os.path.abspath(__file__),
+  "configs_directory": os.path.join(os.path.dirname(os.path.abspath(__file__)), "sample_junit_configs")
+}
+
+
+def setup_suite():
+  if os.path.isdir("/tmp/test_with_naarad"):
+    shutil.rmtree("/tmp/test_with_naarad")
+  if not os.path.isdir(LOGS_DIRECTORY):
+    os.makedirs(LOGS_DIRECTORY)
+  if not os.path.isdir(OUTPUT_DIRECTORY):
+    os.makedirs(OUTPUT_DIRECTORY)
+  global __ssh__
+  __ssh__ = remote_host_helper.sshclient()
+  __ssh__.load_system_host_keys()
+  __ssh__.connect("localhost")
+  sample_code = os.path.join(TEST_DIRECTORY, "samples/trivial_program_with_timing")
+  global __sample_code_in__
+  __sample_code_in__, stdout, stderr = __ssh__.exec_command("python {0}".format(sample_code))
+
+
+def teardown_suite():
+  if __sample_code_in__ is not None:
+    __sample_code_in__.write("quit\n")
+    __sample_code_in__.flush()
+  if __ssh__ is not None:
+    ftp = __ssh__.open_sftp()
+    input_csv = os.path.join(LOGS_DIRECTORY, "output.csv")
+    input_log = os.path.join(LOGS_DIRECTORY, "output.log")
+    ftp.get("/tmp/trivial_timed_output.csv", input_csv)
+    ftp.get("/tmp/trivial_timed_output", input_log)
+    ftp.remove("/tmp/trivial_timed_output.csv")
+    ftp.remove("/tmp/trivial_timed_output")
+    ftp.close()
+    __ssh__.close()
+
+
+def test0():
+  """
+  Yay! Docstring!
+  """
+  assert 1 == 2
+
+
+def test1():
+  __sample_code_in__.write("test1\n")
+  __sample_code_in__.flush()
+  sleep(5)
+  with open("/tmp/trivial_timed_output", "r") as f:
+    lines = f.readlines()
+    test_str = "".join(lines)
+    assert "test1" in test_str
+
+
+def test2():
+  """
+  Another docstring
+  """
+  for i in xrange(50, 150):
+    __sample_code_in__.write("{0}\n".format(i))
+    __sample_code_in__.flush()
+    sleep(.05)
+
+
+def test3():
+    assert 1 == 1
+
+
+def validate2():
+  assert 1 == 1
+
+
+def naarad_config():
+  return os.path.join(TEST_DIRECTORY, "samples/naarad_config.cfg")
+
+
+class MockConfig(object):
+  """
+  Mock config class currently only has a name so that I can use it in execute run and an empty config map
+  """
+  def __init__(self, name):
+    self.name = name
+    self.mapping = {}
+

test/test_test_runner.py

@@ -40,6 +40,16 @@ def test_full_run(self):
     test_runner = TestRunner(test_file, ["test0", "test1", "test2"], {})
     test_runner.run()
 
+  def test_full_run_junit(self):
+    """
+    Tests a full run
+    """
+    runtime.reset_collector()
+    test_file = os.path.join(self.FILE_LOCATION,
+                             "samples/sample_test_junit_reports.py")
+    test_runner = TestRunner(test_file, ["test0", "test1", "test2"], {})
+    test_runner.run()
+
   def test_full_run_parallel(self):
     """
     Tests a full run with parallel tests

zopkio/reporter.py → zopkio/html_reporter.py

@@ -1,4 +1,4 @@
-# Copyright 2014 LinkedIn Corp.
+# Copyright 2015 LinkedIn Corp.
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

zopkio/junit_reporter.py

@@ -0,0 +1,111 @@
+# Copyright 2015 LinkedIn Corp.
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Class used to generate the report.
+"""
+import os
+from jinja2 import Environment, FileSystemLoader
+
+import zopkio.constants as constants
+import zopkio.runtime as runtime
+import zopkio.utils as utils
+from junit_xml import TestSuite, TestCase
+
+
+class _ReportInfo(object):
+  """
+  Holds data shared among all report pages
+  """
+  def __init__(self, output_dir, logs_dir, naarad_dir):
+    self.output_dir = os.path.abspath(output_dir)
+    self.resource_dir = os.path.join(output_dir, "resources/")
+    self.logs_dir = os.path.abspath(logs_dir)
+    self.naarad_dir = os.path.abspath(naarad_dir)
+
+    self.config_to_test_names_map = {}
+    self.junit_xml_path = output_dir
+
+    self.results_map = {
+        "passed": constants.PASSED,
+        "failed": constants.FAILED,
+        "skipped": constants.SKIPPED
+    }
+
+
+class Reporter(object):
+  """
+  Class that converts the aggregated output into a user-friendly web page.
+  """
+  def __init__(self, report_name, output_dir, logs_dir, naarad_dir):
+    """
+    :param report_name: used in the title of the front-end
+    :param output_dir: directory where the report will be generated
+    :param logs_dir: directory of where the logs will be collected
+    :param naarad_dir: directory containing the naarad reports
+    """
+    self.name = report_name
+    self.env = Environment(loader=FileSystemLoader(constants.WEB_RESOURCE_DIR))  # used to load html pages for Jinja2
+    self.data_source = runtime.get_collector()
+    self.report_info = _ReportInfo(output_dir, logs_dir, naarad_dir)
+
+  def get_config_to_test_names_map(self):
+    config_to_test_names_map = {}
+    for config_name in self.data_source.get_config_names():
+      config_to_test_names_map[config_name] = self.data_source.get_test_names(config_name)
+    return config_to_test_names_map
+
+  def get_report_location(self):
+    """
+    Returns the filename of the landing page
+    """
+    return os.path.join(self.report_info.junit_xml_path, 'zopkio_junit_reports.xml')
+
+  def generate(self):
+    """
+    Generates the report
+    """
+    self._setup()
+    for config_name in self.report_info.config_to_test_names_map.keys():
+      config_dir = os.path.join(self.report_info.resource_dir, config_name)
+      utils.makedirs(config_dir)
+      testsuite = self._generate_junit_xml(config_name)
+      with open(os.path.join(self.report_info.junit_xml_path, 'zopkio_junit_reports.xml'), 'w') as file:
+          TestSuite.to_file(file, [testsuite], prettyprint=False)
+
+  def _generate_junit_xml(self, config_name):
+      testcases = []
+      tests=self.data_source.get_test_results(config_name)
+      for test in tests:
+          test_time = 0
+          if test.func_end_time != None and test.func_start_time != None:
+              test_time = test.func_end_time - test.func_start_time
+          tc = TestCase(test.name,config_name,test_time, test.description, test.message)
+          if 'failed' in test.result:
+              tc.add_failure_info(test.result)
+          elif 'skipped' in test.result:
+              tc.add_skipped_info(test.result)
+          testcases.append(tc)
+      testsuite = TestSuite(config_name+'_'+self.name, testcases)
+      return testsuite
+
+  def _setup(self):
+    utils.makedirs(self.report_info.output_dir)
+    utils.makedirs(self.report_info.resource_dir)
+    self.report_info.config_to_test_names_map = self.get_config_to_test_names_map()

zopkio/test_runner.py

@@ -32,7 +32,7 @@
 
 import zopkio.constants as constants
 import zopkio.error_messages as error_messages
-from zopkio.reporter import Reporter
+from zopkio import html_reporter, junit_reporter
 import zopkio.runtime as runtime
 import zopkio.test_runner_helper as test_runner_helper
 import zopkio.utils as utils
@@ -227,7 +227,7 @@ def run(self):
     # analysis.generate_diff_reports()
     self.reporter.data_source.end_time = time.time()
     self.reporter.generate()
-    if not self.master_config.mapping.get("no-display", False):
+    if not self.master_config.mapping.get("no-display", False) and not  self.master_config.mapping.get("junit_reporter", False):
       self._display_results()
 
   def _convert_naarad_slas_to_list(self, naarad_sla_obj):
@@ -562,8 +562,16 @@ def _get_reporter(self):
 
     :return:
     """
-    reporter = Reporter(self.directory_info["report_name"], self.directory_info["results_dir"],
-                        self.directory_info["logs_dir"], self._output_dir)
+    junit_xml_reporter =  self.master_config.mapping.get("junit_reporter", False)
+
+    # default to html
+    if junit_xml_reporter is False:
+      reporter = html_reporter.Reporter(self.directory_info["report_name"], self.directory_info["results_dir"],
+                          self.directory_info["logs_dir"], self._output_dir)
+    else:
+      reporter = junit_reporter.Reporter(self.directory_info["report_name"], self.directory_info["results_dir"],
+                          self.directory_info["logs_dir"], self._output_dir)
+
     return reporter
 
   def _log_results(self, tests):