Skip to content

Commit

Permalink
Adding removal of skipped tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@virenv
virenv committed Jan 8, 2017
1 parent a7e1933 commit 99f4729
Show file tree
Hide file tree
Showing 16 changed files with 269 additions and 1,329 deletions.
3 changes: 3 additions & 0 deletions bin/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
/CustomAnnotations/
/Listeners/
/Tests/
Binary file modified bin/Listeners/RetryAnalyzer.class
View file
Binary file not shown.
9 changes: 9 additions & 0 deletions src/Listeners/RetryAnalyzer.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package Listeners;

import org.testng.IRetryAnalyzer;
import org.testng.ITestContext;
import org.testng.ITestResult;

import CustomAnnotations.RetryCountIfFailed;
Expand Down Expand Up @@ -28,6 +29,14 @@ public boolean retry(ITestResult result) {
// check if the test method had RetryCountIfFailed annotation
RetryCountIfFailed annotation = result.getMethod().getConstructorOrMethod().getMethod()
.getAnnotation(RetryCountIfFailed.class);

// All tests that are retried after failures will appear in the skipped tests
// list. Which causes TestNg to report those retry attempts as skipped tests.
// Here we will explicitly remove the retry test from the skipped tests list so that
// TestNg doesn't report retry attempts as skipped attempts.
// The line below simply does that.
result.getTestContext().getSkippedTests().removeResult(result.getMethod());

// based on the value of annotation see if test needs to be rerun
if((annotation != null) && (counter < annotation.value()))
{
Expand Down
258 changes: 123 additions & 135 deletions test-output/RetryFailedTests/RetryMulitple.html
View file

Large diffs are not rendered by default.

24 changes: 12 additions & 12 deletions test-output/RetryFailedTests/RetryMulitple.xml
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Generated by org.testng.reporters.JUnitXMLReporter -->
<testsuite hostname="Ninja" name="RetryMulitple" tests="12" failures="2" timestamp="13 Dec 2016 18:05:22 GMT" time="36.034" errors="0">
<testcase name="Test1" time="0.017" classname="Tests.Test001">
<testsuite hostname="Ninja" name="RetryMulitple" tests="12" failures="2" timestamp="8 Jan 2017 08:19:20 GMT" time="0.097" errors="0">
<testcase name="Test1" time="0.012" classname="Tests.Test001">
<skipped/>
</testcase> <!-- Test1 -->
<testcase name="Test1" time="0.003" classname="Tests.Test001">
<testcase name="Test1" time="0.002" classname="Tests.Test001">
<skipped/>
</testcase> <!-- Test1 -->
<testcase name="Test1" time="0.003" classname="Tests.Test001">
<testcase name="Test1" time="0.002" classname="Tests.Test001">
<skipped/>
</testcase> <!-- Test1 -->
<testcase name="Test1" time="0.003" classname="Tests.Test001">
<testcase name="Test1" time="0.002" classname="Tests.Test001">
<skipped/>
</testcase> <!-- Test1 -->
<testcase name="Test1" time="0.003" classname="Tests.Test001">
<testcase name="Test1" time="0.002" classname="Tests.Test001">
<skipped/>
</testcase> <!-- Test1 -->
<testcase name="Test1" time="0.002" classname="Tests.Test001">
Expand All @@ -22,27 +22,27 @@
<testcase name="Test1" time="0.002" classname="Tests.Test001">
<skipped/>
</testcase> <!-- Test1 -->
<testcase name="Test1" time="0.005" classname="Tests.Test001">
<testcase name="Test1" time="0.002" classname="Tests.Test001">
<skipped/>
</testcase> <!-- Test1 -->
<testcase name="Test1" time="0.002" classname="Tests.Test001">
<skipped/>
</testcase> <!-- Test1 -->
<testcase name="Test1" time="0.004" classname="Tests.Test001">
<testcase name="Test1" time="0.002" classname="Tests.Test001">
<skipped/>
</testcase> <!-- Test1 -->
<testcase name="Test1" time="0.002" classname="Tests.Test001">
<testcase name="Test1" time="0.001" classname="Tests.Test001">
<failure type="java.lang.AssertionError" message="expected [true] but found [false]">
<![CDATA[java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test1(Test001.java:14)
... Removed 31 stack frames]]>
... Removed 30 stack frames]]>
</failure>
</testcase> <!-- Test1 -->
<testcase name="Test2" time="0.002" classname="Tests.Test001">
<testcase name="Test2" time="0.001" classname="Tests.Test001">
<failure type="java.lang.AssertionError" message="expected [true] but found [false]">
<![CDATA[java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test2(Test001.java:20)
... Removed 31 stack frames]]>
... Removed 30 stack frames]]>
</failure>
</testcase> <!-- Test2 -->
</testsuite> <!-- RetryMulitple -->
5 changes: 5 additions & 0 deletions test-output/RetryFailedTests/testng-failed.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@
<listeners>
<listener class-name="Listeners.AnnotationTransformer"/>
</listeners>
<test name="RetryMulitple">
<classes>
<class name="Tests.Test001"/>
</classes>
</test> <!-- RetryMulitple -->
<test name="RetryMulitple(failed)">
<classes>
<class name="Tests.Test001">
Expand Down
26 changes: 3 additions & 23 deletions test-output/emailable-report.html
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,6 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml"><head><title>TestNG Report</title><style type="text/css">table {margin-bottom:10px;border-collapse:collapse;empty-cells:show}th,td {border:1px solid #009;padding:.25em .5em}th {vertical-align:bottom}td {vertical-align:top}table a {font-weight:bold}.stripe td {background-color: #E6EBF9}.num {text-align:right}.passedodd td {background-color: #3F3}.passedeven td {background-color: #0A0}.skippedodd td {background-color: #DDD}.skippedeven td {background-color: #CCC}.failedodd td,.attn {background-color: #F33}.failedeven td,.stripe .attn {background-color: #D00}.stacktrace {white-space:pre;font-family:monospace}.totop {font-size:85%;text-align:center;border-bottom:2px solid #000}</style></head><body><table><tr><th>Test</th><th># Passed</th><th># Skipped</th><th># Failed</th><th>Time (ms)</th><th>Included Groups</th><th>Excluded Groups</th></tr><tr><th colspan="7">RetryFailedTests</th></tr><tr><td><a href="#t0">RetryMulitple</a></td><td class="num">0</td><td class="num attn">10</td><td class="num attn">2</td><td class="num">36,034</td><td></td><td></td></tr></table><table><thead><tr><th>Class</th><th>Method</th><th>Start</th><th>Time (ms)</th></tr></thead><tbody><tr><th colspan="4">RetryFailedTests</th></tr></tbody><tbody id="t0"><tr><th colspan="4">RetryMulitple &#8212; failed</th></tr><tr class="failedeven"><td rowspan="2">Tests.Test001</td><td><a href="#m0">Test1</a></td><td rowspan="1">1481652322413</td><td rowspan="1">2</td></tr><tr class="failedeven"><td><a href="#m1">Test2</a></td><td rowspan="1">1481652322417</td><td rowspan="1">2</td></tr><tr><th colspan="4">RetryMulitple &#8212; skipped</th></tr><tr class="skippedeven"><td rowspan="10">Tests.Test001</td><td><a href="#m2">Test1</a></td><td rowspan="10">1481652322380</td><td rowspan="10">2</td></tr><tr class="skippedeven"><td><a href="#m3">Test1</a></td></tr><tr class="skippedeven"><td><a href="#m4">Test1</a></td></tr><tr class="skippedeven"><td><a href="#m5">Test1</a></td></tr><tr class="skippedeven"><td><a href="#m6">Test1</a></td></tr><tr class="skippedeven"><td><a href="#m7">Test1</a></td></tr><tr class="skippedeven"><td><a href="#m8">Test1</a></td></tr><tr class="skippedeven"><td><a href="#m9">Test1</a></td></tr><tr class="skippedeven"><td><a href="#m10">Test1</a></td></tr><tr class="skippedeven"><td><a href="#m11">Test1</a></td></tr></tbody></table><h2>RetryMulitple</h2><h3 id="m0">Tests.Test001#Test1</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
<html xmlns="http://www.w3.org/1999/xhtml"><head><title>TestNG Report</title><style type="text/css">table {margin-bottom:10px;border-collapse:collapse;empty-cells:show}th,td {border:1px solid #009;padding:.25em .5em}th {vertical-align:bottom}td {vertical-align:top}table a {font-weight:bold}.stripe td {background-color: #E6EBF9}.num {text-align:right}.passedodd td {background-color: #3F3}.passedeven td {background-color: #0A0}.skippedodd td {background-color: #DDD}.skippedeven td {background-color: #CCC}.failedodd td,.attn {background-color: #F33}.failedeven td,.stripe .attn {background-color: #D00}.stacktrace {white-space:pre;font-family:monospace}.totop {font-size:85%;text-align:center;border-bottom:2px solid #000}</style></head><body><table><tr><th>Test</th><th># Passed</th><th># Skipped</th><th># Failed</th><th>Time (ms)</th><th>Included Groups</th><th>Excluded Groups</th></tr><tr><th colspan="7">RetryFailedTests</th></tr><tr><td><a href="#t0">RetryMulitple</a></td><td class="num">0</td><td class="num">0</td><td class="num attn">2</td><td class="num">97</td><td></td><td></td></tr></table><table><thead><tr><th>Class</th><th>Method</th><th>Start</th><th>Time (ms)</th></tr></thead><tbody><tr><th colspan="4">RetryFailedTests</th></tr></tbody><tbody id="t0"><tr><th colspan="4">RetryMulitple &#8212; failed</th></tr><tr class="failedeven"><td rowspan="2">Tests.Test001</td><td><a href="#m0">Test1</a></td><td rowspan="1">1483863560537</td><td rowspan="1">1</td></tr><tr class="failedeven"><td><a href="#m1">Test2</a></td><td rowspan="1">1483863560543</td><td rowspan="1">1</td></tr></tbody></table><h2>RetryMulitple</h2><h3 id="m0">Tests.Test001#Test1</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test1(Test001.java:14)
... Removed 31 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p><h3 id="m1">Tests.Test001#Test2</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
... Removed 30 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p><h3 id="m1">Tests.Test001#Test2</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test2(Test001.java:20)
... Removed 31 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p><h3 id="m2">Tests.Test001#Test1</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test1(Test001.java:14)
... Removed 31 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p><h3 id="m3">Tests.Test001#Test1</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test1(Test001.java:14)
... Removed 31 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p><h3 id="m4">Tests.Test001#Test1</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test1(Test001.java:14)
... Removed 31 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p><h3 id="m5">Tests.Test001#Test1</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test1(Test001.java:14)
... Removed 31 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p><h3 id="m6">Tests.Test001#Test1</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test1(Test001.java:14)
... Removed 31 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p><h3 id="m7">Tests.Test001#Test1</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test1(Test001.java:14)
... Removed 31 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p><h3 id="m8">Tests.Test001#Test1</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test1(Test001.java:14)
... Removed 31 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p><h3 id="m9">Tests.Test001#Test1</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test1(Test001.java:14)
... Removed 31 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p><h3 id="m10">Tests.Test001#Test1</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test1(Test001.java:14)
... Removed 31 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p><h3 id="m11">Tests.Test001#Test1</h3><table class="result"><tr><th>Exception</th></tr><tr><td><div class="stacktrace">java.lang.AssertionError: expected [true] but found [false]
at Tests.Test001.Test1(Test001.java:14)
... Removed 31 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p></body></html>
... Removed 30 stack frames</div></td></tr></table><p class="totop"><a href="#summary">back to summary</a></p></body></html>
Loading

0 comments on commit 99f4729

Please sign in to comment.