Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable Dataprovider failures to be considered. #2748

Merged
merged 1 commit into from
Apr 13, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,5 @@ test-output-tests
testng.iml
z_build
.DS_Store

outputDir/
**/Version.java
1 change: 1 addition & 0 deletions CHANGES.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
Current
7.6.0
Fixed: GITHUB-217: Configure TestNG to fail when there's a failure in data provider (Krishnan Mahadevan)
Fixed: GITHUB-2743: SuiteRunner could not be initial by default Configuration (Nan Liang)
Fixed: GITHUB-2729: beforeConfiguration() listener method should be invoked for skipped configurations as well(Nan Liang)
Fixed: assertEqualsNoOrder for Collection and Iterators size check was missing (Adam Kaczmarek)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,9 @@ public interface IDataProviderMethod {

/** @return Which indices to run from this data provider, default: all. */
List<Integer> getIndices();

/** @return Whether failures in data providers should be treated as test failures */
default boolean propagateFailureAsTestFailure() {
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,4 +48,11 @@
* @return the value
*/
int[] indices() default {};

/**
* Helps TestNG decide if it should treat data provider failures as test failures.
*
* @return the value
*/
boolean propagateFailureAsTestFailure() default false;
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,10 @@ public interface IDataProviderAnnotation extends IAnnotation {
List<Integer> getIndices();

void setIndices(List<Integer> indices);

/** Have TestNG consider failures in data provider methods as test failures. */
void propagateFailureAsTestFailure();

/** @return - <code>true</code>If data provider failures should be propagated as test failures */
boolean isPropagateFailureAsTestFailure();
}
8 changes: 8 additions & 0 deletions testng-core/src/main/java/org/testng/CommandLineArgs.java
Original file line number Diff line number Diff line change
Expand Up @@ -251,4 +251,12 @@ public class CommandLineArgs {
description =
"Should TestNG report all iterations of a data driven test as individual skips, in-case of upstream failures.")
public Boolean includeAllDataDrivenTestsWhenSkipping = false;

public static final String PROPAGATE_DATA_PROVIDER_FAILURES_AS_TEST_FAILURE =
"-propagateDataProviderFailureAsTestFailure";

@Parameter(
names = PROPAGATE_DATA_PROVIDER_FAILURES_AS_TEST_FAILURE,
description = "Should TestNG consider failures in Data Providers as test failures.")
public Boolean propagateDataProviderFailureAsTestFailure = false;
}
11 changes: 11 additions & 0 deletions testng-core/src/main/java/org/testng/TestNG.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;
Expand Down Expand Up @@ -596,6 +597,14 @@ public boolean getReportAllDataDrivenTestsAsSkipped() {
return this.m_configuration.getReportAllDataDrivenTestsAsSkipped();
}

public void propagateDataProviderFailureAsTestFailure() {
this.m_configuration.propagateDataProviderFailureAsTestFailure();
}

public boolean isPropagateDataProviderFailureAsTestFailure() {
return this.m_configuration.isPropagateDataProviderFailureAsTestFailure();
}

/**
* Set the suites file names to be run by this TestNG object. This method tries to load and parse
* the specified TestNG suite xml files. If a file is missing, it is ignored.
Expand Down Expand Up @@ -1421,6 +1430,8 @@ public static TestNG privateMain(String[] argv, ITestListener listener) {
* @param cla The command line parameters
*/
protected void configure(CommandLineArgs cla) {
Optional.ofNullable(cla.propagateDataProviderFailureAsTestFailure)
.ifPresent(value -> propagateDataProviderFailureAsTestFailure());
setReportAllDataDrivenTestsAsSkipped(cla.includeAllDataDrivenTestsWhenSkipping);
if (cla.verbose != null) {
setVerbose(cla.verbose);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ public class Configuration implements IConfiguration {

private boolean includeAllDataDrivenTestsWhenSkipping;

private boolean propagateDataProviderFailureAsTestFailure;

public Configuration() {
init(new JDK15AnnotationFinder(new DefaultAnnotationTransformer()));
}
Expand Down Expand Up @@ -156,4 +158,14 @@ public void setReportAllDataDrivenTestsAsSkipped(boolean reportAllDataDrivenTest
public boolean getReportAllDataDrivenTestsAsSkipped() {
return this.includeAllDataDrivenTestsWhenSkipping;
}

@Override
public void propagateDataProviderFailureAsTestFailure() {
this.propagateDataProviderFailureAsTestFailure = true;
}

@Override
public boolean isPropagateDataProviderFailureAsTestFailure() {
return propagateDataProviderFailureAsTestFailure;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,9 @@ public boolean isParallel() {
public List<Integer> getIndices() {
return annotation.getIndices();
}

@Override
public boolean propagateFailureAsTestFailure() {
return annotation.isPropagateFailureAsTestFailure();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -55,4 +55,8 @@ default void setReportAllDataDrivenTestsAsSkipped(boolean reportAllDataDrivenTes
default boolean getReportAllDataDrivenTestsAsSkipped() {
return false;
}

void propagateDataProviderFailureAsTestFailure();

boolean isPropagateDataProviderFailureAsTestFailure();
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ public class DataProviderAnnotation extends BaseAnnotation implements IDataProvi
private String m_name;
private boolean m_parallel;
private List<Integer> m_indices;
private boolean m_bubbleUpFailures = false;

@Override
public boolean isParallel() {
Expand Down Expand Up @@ -39,4 +40,14 @@ public List<Integer> getIndices() {
public void setIndices(List<Integer> indices) {
m_indices = indices;
}

@Override
public void propagateFailureAsTestFailure() {
m_bubbleUpFailures = true;
}

@Override
public boolean isPropagateFailureAsTestFailure() {
return m_bubbleUpFailures;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,9 @@ private IAnnotation createDataProviderTag(Method method, Annotation a) {
}
result.setParallel(c.parallel());
result.setIndices(Ints.asList(c.indices()));
if (c.propagateFailureAsTestFailure()) {
result.propagateFailureAsTestFailure();
}

return result;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
import static org.testng.internal.Parameters.MethodParameters;

import java.util.Map;
import java.util.Optional;
import org.testng.DataProviderHolder;
import org.testng.IDataProviderMethod;
import org.testng.ITestContext;
import org.testng.ITestNGMethod;
import org.testng.ITestObjectFactory;
Expand Down Expand Up @@ -88,7 +90,12 @@ private ParameterBag handleParameters(
}

ITestResult result = TestResult.newTestResultWithCauseAs(testMethod, testContext, cause);
return new ParameterBag(result);

boolean bubbleUpFailure =
Optional.ofNullable(testMethod.getDataProviderMethod())
.map(IDataProviderMethod::propagateFailureAsTestFailure)
.orElse(false);
return new ParameterBag(result, bubbleUpFailure);
}
}

Expand All @@ -99,15 +106,17 @@ private ParameterBag handleParameters(
static class ParameterBag {
final ParameterHolder parameterHolder;
final ITestResult errorResult;
boolean bubbleUpFailures = false;
krmahadevan marked this conversation as resolved.
Show resolved Hide resolved

ParameterBag(ParameterHolder parameterHolder) {
this.parameterHolder = parameterHolder;
this.errorResult = null;
}

ParameterBag(ITestResult errorResult) {
ParameterBag(ITestResult errorResult, boolean bubbleUpFailures) {
this.parameterHolder = null;
this.errorResult = errorResult;
this.bubbleUpFailures = bubbleUpFailures;
}

boolean hasErrors() {
Expand All @@ -119,5 +128,9 @@ boolean runInParallel() {
&& (parameterHolder.origin == ParameterHolder.ParameterOrigin.ORIGIN_DATA_PROVIDER
&& parameterHolder.dataProviderHolder.isParallel()));
}

boolean isBubbleUpFailures() {
return bubbleUpFailures;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -924,7 +924,11 @@ public int invoke(int invCount) {
if (bag.hasErrors()) {
ITestResult tr = bag.errorResult;
Throwable throwable = Objects.requireNonNull(tr).getThrowable();
if (throwable instanceof TestNGException) {
boolean bubbleUpFailures =
m_configuration.isPropagateDataProviderFailureAsTestFailure()
|| bag.isBubbleUpFailures();

if (throwable instanceof TestNGException || bubbleUpFailures) {
tr.setStatus(ITestResult.FAILURE);
m_notifier.addFailedTest(arguments.getTestMethod(), tr);
} else {
Expand Down
105 changes: 105 additions & 0 deletions testng-core/src/test/java/test/dataprovider/DataProviderTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,19 @@
import java.util.Collections;
import java.util.List;
import org.assertj.core.api.Condition;
import org.assertj.core.api.SoftAssertions;
import org.testng.Assert;
import org.testng.IDataProviderMethod;
import org.testng.ITestNGMethod;
import org.testng.ITestResult;
import org.testng.TestListenerAdapter;
import org.testng.TestNG;
import org.testng.TestNGException;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import org.testng.internal.reflect.MethodMatcherException;
import org.testng.xml.XmlClass;
import org.testng.xml.XmlTest;
import test.InvokedMethodNameListener;
import test.SimpleBaseTest;
import test.dataprovider.issue1691.DataProviderDefinitionAtClassLevelAndNoTestMethodUsage;
Expand All @@ -23,6 +29,12 @@
import test.dataprovider.issue1691.withinheritance.ChildClassHasFullDefinitionOfDataProviderAtClassLevel;
import test.dataprovider.issue1691.withinheritance.ChildClassHasPartialDefinitionOfDataProviderAtClassLevel;
import test.dataprovider.issue1691.withinheritance.ChildClassWithNoDataProviderInformationInTestMethod;
import test.dataprovider.issue1987.BaseClassSample;
import test.dataprovider.issue1987.DataProviderInBaseClassSample;
import test.dataprovider.issue1987.DataProviderInDifferentClassSample;
import test.dataprovider.issue1987.DataProviderInSameClassSample;
import test.dataprovider.issue1987.DataProviderTrackingListener;
import test.dataprovider.issue2504.SampleTestCaseListener;
import test.dataprovider.issue2565.Data;
import test.dataprovider.issue2565.SampleTestUsingConsumer;
import test.dataprovider.issue2565.SampleTestUsingFunction;
Expand Down Expand Up @@ -406,4 +418,97 @@ public void retryWithDataProvider() {
assertThat(tla.getFailedTests()).size().isEqualTo(1);
assertThat(tla.getSkippedTests()).size().isEqualTo(2);
}

@Test(description = "GITHUB-217", expectedExceptions = TestNGException.class)
public void ensureTestNGThrowsExceptionWhenAllTestsAreSkipped() {
TestNG testng = create(test.dataprovider.issue217.TestClassSample.class);
testng.toggleFailureIfAllTestsWereSkipped(true);
testng.run();
}

@Test(description = "GITHUB-217")
public void ensureTestNGFailsDueToDataProviderFailure() {
TestNG testng = create(test.dataprovider.issue217.TestClassSample.class);
testng.propagateDataProviderFailureAsTestFailure();
testng.run();
assertThat(testng.getStatus()).isEqualTo(1);
}

@Test(description = "GITHUB-217")
public void ensureTestNGFailsDueToDataProviderFailure2() {
TestNG testng = create(test.dataprovider.issue217.AnotherTestClassSample.class);
testng.run();
assertThat(testng.getStatus()).isEqualTo(1);
}

@Test(description = "GITHUB-2255")
public void ensureDataProviderValuesAreVisibleToConfigMethods() {
TestNG testNG = create(test.dataprovider.issue2255.TestClassSample.class);
testNG.run();
assertThat(test.dataprovider.issue2255.TestClassSample.data).containsExactly(100, 200);
}

@Test(dataProvider = "testData", description = "GITHUB-1987")
public void extractDataProviderInfoWhenDpResidesInSameClass(
Class<?> clazz, boolean performInstanceCheck, Class<?> dataProviderClass) {
TestNG testng = create(clazz);
DataProviderTrackingListener listener = new DataProviderTrackingListener();
testng.addListener(listener);
testng.run();
ITestNGMethod method = listener.getResult().getMethod();
IDataProviderMethod dpm = method.getDataProviderMethod();
assertThat(dpm).isNotNull();
if (performInstanceCheck) {
assertThat(dpm.getInstance()).isEqualTo(method.getInstance());
}
assertThat(dpm.getMethod().getName()).isEqualTo("getData");
assertThat(dpm.getInstance().getClass()).isEqualTo(dataProviderClass);
}

@DataProvider(name = "testData")
public Object[][] getTestData() {
return new Object[][] {
{DataProviderInSameClassSample.class, true, DataProviderInSameClassSample.class},
{DataProviderInBaseClassSample.class, true, DataProviderInBaseClassSample.class},
{DataProviderInDifferentClassSample.class, false, BaseClassSample.class}
};
}

@Test(description = "GITHUB-2267")
public void ensureDynamicRetryAnalyzersAreHonouredForDataDrivenTest() {
TestNG testng = create(test.dataprovider.issue2267.TestClassSample.class);
TestListenerAdapter tla = new TestListenerAdapter();
testng.addListener(tla);
testng.run();
assertThat(tla.getFailedTests()).size().isEqualTo(1);
assertThat(tla.getSkippedTests()).size().isEqualTo(1);
}

@Test(description = "GITHUB-2327")
public void ensureDataProviderParametersAreAlwaysAvailableForListeners() {
TestNG testng = create(test.dataprovider.issue2327.TestClassSample.class);
TestListenerAdapter tla = new TestListenerAdapter();
testng.addListener(tla);
testng.run();

assertThat(tla.getSkippedTests().size()).isEqualTo(2);
SoftAssertions assertions = new SoftAssertions();

for (ITestResult skippedTest : tla.getSkippedTests()) {
assertions.assertThat(skippedTest.getParameters()).isNotEmpty();
}
assertions.assertAll();
}

@Test(description = "GITHUB-2504")
public void ensureParametersCopiedOnConfigFailures() {
XmlTest xmltest = createXmlTest("2504_suite", "2504_test");
xmltest.setXmlClasses(
Collections.singletonList(new XmlClass(test.dataprovider.issue2504.TestClassSample.class)));
TestNG testNG = create(Collections.singletonList(xmltest.getSuite()));
SampleTestCaseListener listener = new SampleTestCaseListener();
testNG.addListener(listener);
testNG.run();
assertThat(listener.getParameters()).containsExactlyElementsOf(Arrays.asList(1, 2, 3, 4, 5));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import org.testng.annotations.Test;

public class DataProviderInBaseClass extends BaseClassSample {
public class DataProviderInBaseClassSample extends BaseClassSample {

@Test(dataProvider = "dp")
public void testMethod(int i) {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import org.testng.annotations.Test;

public class DataProviderInDifferentClass {
public class DataProviderInDifferentClassSample {

@Test(dataProvider = "dp", dataProviderClass = BaseClassSample.class)
public void testMethod(int i) {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

public class DataProviderInSameClass {
public class DataProviderInSameClassSample {

@Test(dataProvider = "dp")
public void testMethod(int i) {}
Expand Down
Loading