SONARPY-263 Import unit tests execution reports generated by py.test

its/plugin/projects/nosetests_project/pytest.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<testsuite errors="0" failures="3" name="pytest" skips="0" tests="4" time="0.035">
+  <testcase classname="tests.test_prod" file="tests/test_prod.py" line="0" name="test_succeeding" time="0.0007090568542480469"></testcase>
+  <testcase classname="tests.test_prod" file="tests/test_prod.py" line="3" name="test_failing" time="0.0010173320770263672">
+    <failure message="assert False">def test_failing():
+      &gt; assert False
+      E assert False
+
+      tests/test_prod.py:5: AssertionError
+    </failure>
+  </testcase>
+  <testcase classname="tests.test_prod" file="tests/test_prod.py" line="6" name="test_crashing" time="0.0005733966827392578">
+    <failure message="RuntimeError">def test_crashing():
+      &gt; raise RuntimeError()
+      E RuntimeError
+
+      tests/test_prod.py:8: RuntimeError
+    </failure>
+  </testcase>
+  <testcase classname="tests.test_prod" file="tests/test_prod.py" line="9" name="test_skipped" time="0.0007612705230712891">
+    <skipped message="skipped message" type="pytest.skip">skipped</skipped>
+  </testcase>
+</testsuite>

its/plugin/src/test/java/com/sonar/python/it/plugin/TestReportTest.java

@@ -69,6 +69,19 @@ public void import_report() throws Exception {
       .build());
   }
 
+  @Test
+  public void import_pytest_report() {
+    orchestrator.executeBuild(createBuild("pytest.xml"));
+    assertProjectMeasures(PROJECT, new ImmutableMap.Builder<String, Integer>()
+      .put("tests", 3)
+      .put("test_failures", 2)
+      .put("test_errors", 0)
+      .put("skipped_tests", 1)
+      .put("test_success_density", 33)
+      .put("test_execution_time", 1)
+      .build());
+  }
+
   @Test
   public void simple_mode() throws Exception {
     orchestrator.executeBuild(createBuild("nosetests.xml").setProperty("sonar.python.xunit.skipDetails", "true"));

sonar-python-plugin/src/main/java/org/sonar/plugins/python/xunit/PythonXUnitSensor.java

@@ -24,6 +24,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import javax.annotation.CheckForNull;
 import javax.xml.stream.XMLStreamException;
 import org.apache.commons.lang.StringUtils;
 import org.sonar.api.batch.fs.FileSystem;
@@ -78,26 +79,16 @@ private static void simpleMode(final SensorContext context, List<File> reports)
       parser.parse(report);
     }
 
-    int testsCount = 0;
-    int testsSkipped = 0;
-    int testsErrors = 0;
-    int testsFailures = 0;
-    long testsTime = 0;
-    for (TestSuite report : parserHandler.getParsedReports()) {
-      testsCount += report.getTests() - report.getSkipped();
-      testsSkipped += report.getSkipped();
-      testsErrors += report.getErrors();
-      testsFailures += report.getFailures();
-      testsTime += report.getTime();
-    }
+    TestResult total = new TestResult();
+    parserHandler.getParsedReports().forEach(testSuite -> testSuite.getTestCases().forEach(total::addTestCase));
 
-    if (testsCount > 0) {
+    if (total.getTests() > 0) {
       InputComponent module = context.module();
-      saveMeasure(context, module, CoreMetrics.TESTS, testsCount);
-      saveMeasure(context, module, CoreMetrics.SKIPPED_TESTS, testsSkipped);
-      saveMeasure(context, module, CoreMetrics.TEST_ERRORS, testsErrors);
-      saveMeasure(context, module, CoreMetrics.TEST_FAILURES, testsFailures);
-      saveMeasure(context, module, CoreMetrics.TEST_EXECUTION_TIME, testsTime);
+      saveMeasure(context, module, CoreMetrics.TESTS, total.getExecutedTests());
+      saveMeasure(context, module, CoreMetrics.SKIPPED_TESTS, total.getSkipped());
+      saveMeasure(context, module, CoreMetrics.TEST_ERRORS, total.getErrors());
+      saveMeasure(context, module, CoreMetrics.TEST_FAILURES, total.getFailures());
+      saveMeasure(context, module, CoreMetrics.TEST_EXECUTION_TIME, total.getTime());
     }
   }
 
@@ -114,24 +105,35 @@ private void detailedMode(final SensorContext context, List<File> reports) throw
   }
 
   private void processReportDetailed(SensorContext context, Collection<TestSuite> parsedReports) {
-    Collection<TestSuite> locatedResources = lookupResources(parsedReports);
-    for (TestSuite fileReport : locatedResources) {
-      InputFile inputFile = fileReport.getInputFile();
+    Map<InputFile, TestResult> locatedResources = lookupResources(parsedReports);
+    for (Map.Entry<InputFile, TestResult> entry : locatedResources.entrySet()) {
+      InputFile inputFile = entry.getKey();
+      TestResult fileTestResult = entry.getValue();
+      LOG.debug("Saving test execution measures for '{}'", inputFile.toString());
+
+      saveMeasure(context, inputFile, CoreMetrics.SKIPPED_TESTS, fileTestResult.getSkipped());
+      saveMeasure(context, inputFile, CoreMetrics.TESTS, fileTestResult.getExecutedTests());
+      saveMeasure(context, inputFile, CoreMetrics.TEST_ERRORS, fileTestResult.getErrors());
+      saveMeasure(context, inputFile, CoreMetrics.TEST_FAILURES, fileTestResult.getFailures());
+      saveMeasure(context, inputFile, CoreMetrics.TEST_EXECUTION_TIME, fileTestResult.getTime());
+    }
+  }
 
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Saving test execution measures for '{}' under resource '{}'", fileReport.getKey(), inputFile.relativePath());
-      }
+  @CheckForNull
+  private InputFile findResource(TestCase testCase, String fileKey) {
+    InputFile unitTestFile = null;
 
-      saveMeasure(context, inputFile, CoreMetrics.SKIPPED_TESTS, fileReport.getSkipped());
-      saveMeasure(context, inputFile, CoreMetrics.TESTS, fileReport.getTests() - fileReport.getSkipped());
-      saveMeasure(context, inputFile, CoreMetrics.TEST_ERRORS, fileReport.getErrors());
-      saveMeasure(context, inputFile, CoreMetrics.TEST_FAILURES, fileReport.getFailures());
-      saveMeasure(context, inputFile, CoreMetrics.TEST_EXECUTION_TIME, fileReport.getTime());
+    if (testCase.getFile() != null) {
+      unitTestFile = getSonarTestFile(new File(testCase.getFile()));
     }
-  }
 
-  private InputFile findResource(String fileKey) {
-    return findResourceUsingNoseTestsStrategy(fileKey);
+    if (unitTestFile == null) {
+      String testClassname = testCase.getTestClassname();
+      String key = testClassname != null ? testClassname : fileKey;
+      return findResourceUsingNoseTestsStrategy(key);
+    }
+
+    return unitTestFile;
   }
 
   private InputFile findResourceUsingNoseTestsStrategy(String fileKey) {
@@ -151,32 +153,27 @@ private InputFile findResourceUsingNoseTestsStrategy(String fileKey) {
     return unitTestFile;
   }
 
-  private Collection<TestSuite> lookupResources(Collection<TestSuite> testReports) {
-    Map<String, TestSuite> locatedReports = new HashMap<>();
-
-    for (TestSuite report : testReports) {
-      String fileKey = report.getKey();
-
-      LOG.debug("Trying to find a SonarQube resource for '{}'", fileKey);
-      InputFile inputFile = findResource(fileKey);
-      if (inputFile != null) {
-        LOG.debug("The resource was found '{}'", inputFile);
-
-        TestSuite summaryReport = locatedReports.get(inputFile.absolutePath());
-        if (summaryReport != null) {
-          summaryReport.addMeasures(report);
+  private Map<InputFile, TestResult> lookupResources(Collection<TestSuite> testReports) {
+    Map<InputFile, TestResult> testResultsByFile = new HashMap<>();
+
+    for (TestSuite testSuite : testReports) {
+      testSuite.getTestCases().forEach(testCase -> {
+        String testClassname = testCase.getTestClassname();
+        LOG.debug("Trying to find a SonarQube resource for test case '{}'", testClassname);
+        InputFile inputFile = findResource(testCase, testSuite.getKey());
+        if (inputFile != null) {
+          LOG.debug("The resource was found '{}'", inputFile);
+          testResultsByFile.computeIfAbsent(inputFile, k -> new TestResult()).addTestCase(testCase);
         } else {
-          report.setInputFile(inputFile);
-          locatedReports.put(inputFile.absolutePath(), report);
+          LOG.warn("The resource for '{}' is not found, drilling down to the details of this test won't be possible", testClassname);
         }
-      } else {
-        LOG.warn("The resource for '{}' is not found, drilling down to the details of this test won't be possible", fileKey);
-      }
+      });
     }
 
-    return locatedReports.values();
+    return testResultsByFile;
   }
 
+  @CheckForNull
   private InputFile getSonarTestFile(File file) {
     LOG.debug("Using the key '{}' to lookup the resource in SonarQube", file.getPath());
     return fileSystem.inputFile(fileSystem.predicates().is(file));

sonar-python-plugin/src/main/java/org/sonar/plugins/python/xunit/TestCase.java

@@ -19,6 +19,8 @@
  */
 package org.sonar.plugins.python.xunit;
 
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
 import org.apache.commons.lang.StringEscapeUtils;
 
 /**
@@ -27,20 +29,22 @@
  */
 public class TestCase {
 
-  private static final String STATUS_OK = "ok";
-  private static final String STATUS_ERROR = "error";
-  private static final String STATUS_FAILURE = "failure";
-  private static final String STATUS_SKIPPED = "skipped";
+  public static final String STATUS_OK = "ok";
+  public static final String STATUS_ERROR = "error";
+  public static final String STATUS_FAILURE = "failure";
+  public static final String STATUS_SKIPPED = "skipped";
 
-  private String name;
-  private String status = STATUS_OK;
-  private String stackTrace;
-  private String errorMessage;
-  private int time = 0;
+  private final String name;
+  private final String status;
+  private final String stackTrace;
+  private final String errorMessage;
+  private final int time;
+  private final String file;
+  private final String testClassname;
 
   /**
    * Constructs a testcase instance out of following parameters
-   * 
+   *
    * @param name
    *          The name of this testcase
    * @param time
@@ -51,13 +55,19 @@ public class TestCase {
    *          The stack trace occurred while executing of this testcase; pass "" if the testcase passed/skipped.
    * @param msg
    *          The error message associated with this testcase of the execution was erroneous; pass "" if not.
+   * @param file
+   *          The optional file to which this test case applies.
+   * @param testClassname
+   *          The classname of the test.
    */
-  public TestCase(String name, int time, String status, String stack, String msg) {
+  public TestCase(String name, int time, String status, String stack, String msg, @Nullable String file, @Nullable String testClassname) {
     this.name = name;
     this.time = time;
     this.stackTrace = stack;
     this.errorMessage = msg;
     this.status = status;
+    this.file = file;
+    this.testClassname = testClassname;
   }
 
   /**
@@ -85,6 +95,16 @@ public int getTime() {
     return time;
   }
 
+  @CheckForNull
+  public String getFile() {
+    return file;
+  }
+
+  @CheckForNull
+  public String getTestClassname() {
+    return testClassname;
+  }
+
   /**
    * Returns execution details as sonar-conform XML
    */

sonar-python-plugin/src/main/java/org/sonar/plugins/python/xunit/TestResult.java

@@ -0,0 +1,66 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2019 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.plugins.python.xunit;
+
+public class TestResult {
+
+  private int errors = 0;
+  private int skipped = 0;
+  private int tests = 0;
+  private int time = 0;
+  private int failures = 0;
+
+  public int getErrors() {
+    return errors;
+  }
+
+  public int getSkipped() {
+    return skipped;
+  }
+
+  public int getTests() {
+    return tests;
+  }
+
+  public int getExecutedTests() {
+    return tests - skipped;
+  }
+
+  public int getTime() {
+    return time;
+  }
+
+  public int getFailures() {
+    return failures;
+  }
+
+  public void addTestCase(TestCase tc) {
+    if (tc.isSkipped()) {
+      skipped++;
+    } else if (tc.isFailure()) {
+      failures++;
+    } else if (tc.isError()) {
+      errors++;
+    }
+    tests++;
+    time += tc.getTime();
+  }
+
+}