From b5b3e1d0f8db429b1bb5d1d675ddf5940d19f92c Mon Sep 17 00:00:00 2001 From: Krishnan Mahadevan Date: Tue, 12 Apr 2022 13:39:35 +0530 Subject: [PATCH] Enable Dataprovider failures to be considered. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #217 Enable a data provider to propagate its failure And be considered as a test failure. This can be done at an individual data provider level by using the “@DataProvider” annotation’s new attribute “propagateFailureAsTestFailure” This can be done at the entire TestNG level by using the configuration parameter “-propagateDataProviderFailureAsTestFailure true” Also regrouped the data provider test into single test class. --- .gitignore | 2 +- CHANGES.txt | 1 + .../java/org/testng/IDataProviderMethod.java | 5 + .../org/testng/annotations/DataProvider.java | 7 ++ .../annotations/IDataProviderAnnotation.java | 6 + .../main/java/org/testng/CommandLineArgs.java | 8 ++ .../src/main/java/org/testng/TestNG.java | 11 ++ .../org/testng/internal/Configuration.java | 12 ++ .../testng/internal/DataProviderMethod.java | 5 + .../org/testng/internal/IConfiguration.java | 4 + .../annotations/DataProviderAnnotation.java | 11 ++ .../internal/annotations/JDK15TagFactory.java | 3 + .../internal/invokers/ParameterHandler.java | 17 ++- .../testng/internal/invokers/TestInvoker.java | 6 +- .../test/dataprovider/DataProviderTest.java | 105 ++++++++++++++++++ ...ava => DataProviderInBaseClassSample.java} | 2 +- ...> DataProviderInDifferentClassSample.java} | 2 +- ...ava => DataProviderInSameClassSample.java} | 2 +- .../dataprovider/issue1987/IssueTest.java | 39 ------- .../issue217/AnotherTestClassSample.java | 16 +++ .../test/dataprovider/issue217/IssueTest.java | 16 --- ...mpleTestCase.java => TestClassSample.java} | 2 +- .../dataprovider/issue2255/IssueTest.java | 17 --- ...mpleTestCase.java => TestClassSample.java} | 4 +- .../dataprovider/issue2267/IssueTest.java | 21 ---- ...mpleTestCase.java => TestClassSample.java} | 2 +- .../dataprovider/issue2327/IssueTest.java | 26 ----- ...mpleTestCase.java => TestClassSample.java} | 2 +- .../dataprovider/issue2504/IssueTest.java | 25 ----- ...mpleTestCase.java => TestClassSample.java} | 2 +- testng-core/src/test/resources/testng.xml | 6 - 31 files changed, 224 insertions(+), 163 deletions(-) rename testng-core/src/test/java/test/dataprovider/issue1987/{DataProviderInBaseClass.java => DataProviderInBaseClassSample.java} (67%) rename testng-core/src/test/java/test/dataprovider/issue1987/{DataProviderInDifferentClass.java => DataProviderInDifferentClassSample.java} (78%) rename testng-core/src/test/java/test/dataprovider/issue1987/{DataProviderInSameClass.java => DataProviderInSameClassSample.java} (86%) delete mode 100644 testng-core/src/test/java/test/dataprovider/issue1987/IssueTest.java create mode 100644 testng-core/src/test/java/test/dataprovider/issue217/AnotherTestClassSample.java delete mode 100644 testng-core/src/test/java/test/dataprovider/issue217/IssueTest.java rename testng-core/src/test/java/test/dataprovider/issue217/{SampleTestCase.java => TestClassSample.java} (92%) delete mode 100644 testng-core/src/test/java/test/dataprovider/issue2255/IssueTest.java rename testng-core/src/test/java/test/dataprovider/issue2255/{SampleTestCase.java => TestClassSample.java} (90%) delete mode 100644 testng-core/src/test/java/test/dataprovider/issue2267/IssueTest.java rename testng-core/src/test/java/test/dataprovider/issue2267/{SampleTestCase.java => TestClassSample.java} (90%) delete mode 100644 testng-core/src/test/java/test/dataprovider/issue2327/IssueTest.java rename testng-core/src/test/java/test/dataprovider/issue2327/{SampleTestCase.java => TestClassSample.java} (95%) delete mode 100644 testng-core/src/test/java/test/dataprovider/issue2504/IssueTest.java rename testng-core/src/test/java/test/dataprovider/issue2504/{SampleTestCase.java => TestClassSample.java} (94%) diff --git a/.gitignore b/.gitignore index 412e5342d2..e16cab7532 100644 --- a/.gitignore +++ b/.gitignore @@ -24,5 +24,5 @@ test-output-tests testng.iml z_build .DS_Store - +outputDir/ **/Version.java diff --git a/CHANGES.txt b/CHANGES.txt index 8ffd633ff8..fc9351aeca 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,6 @@ Current 7.6.0 +Fixed: GITHUB-217: Configure TestNG to fail when there's a failure in data provider (Krishnan Mahadevan) Fixed: GITHUB-2743: SuiteRunner could not be initial by default Configuration (Nan Liang) Fixed: GITHUB-2729: beforeConfiguration() listener method should be invoked for skipped configurations as well(Nan Liang) Fixed: assertEqualsNoOrder for Collection and Iterators size check was missing (Adam Kaczmarek) diff --git a/testng-core-api/src/main/java/org/testng/IDataProviderMethod.java b/testng-core-api/src/main/java/org/testng/IDataProviderMethod.java index 7a495bab1d..324f955e3f 100644 --- a/testng-core-api/src/main/java/org/testng/IDataProviderMethod.java +++ b/testng-core-api/src/main/java/org/testng/IDataProviderMethod.java @@ -25,4 +25,9 @@ public interface IDataProviderMethod { /** @return Which indices to run from this data provider, default: all. */ List getIndices(); + + /** @return Whether failures in data providers should be treated as test failures */ + default boolean propagateFailureAsTestFailure() { + return false; + } } diff --git a/testng-core-api/src/main/java/org/testng/annotations/DataProvider.java b/testng-core-api/src/main/java/org/testng/annotations/DataProvider.java index 2d4dfa2bf8..b30dd366ef 100644 --- a/testng-core-api/src/main/java/org/testng/annotations/DataProvider.java +++ b/testng-core-api/src/main/java/org/testng/annotations/DataProvider.java @@ -48,4 +48,11 @@ * @return the value */ int[] indices() default {}; + + /** + * Helps TestNG decide if it should treat data provider failures as test failures. + * + * @return the value + */ + boolean propagateFailureAsTestFailure() default false; } diff --git a/testng-core-api/src/main/java/org/testng/annotations/IDataProviderAnnotation.java b/testng-core-api/src/main/java/org/testng/annotations/IDataProviderAnnotation.java index 79b57bf2df..372b6f24f5 100644 --- a/testng-core-api/src/main/java/org/testng/annotations/IDataProviderAnnotation.java +++ b/testng-core-api/src/main/java/org/testng/annotations/IDataProviderAnnotation.java @@ -21,4 +21,10 @@ public interface IDataProviderAnnotation extends IAnnotation { List getIndices(); void setIndices(List indices); + + /** Have TestNG consider failures in data provider methods as test failures. */ + void propagateFailureAsTestFailure(); + + /** @return - trueIf data provider failures should be propagated as test failures */ + boolean isPropagateFailureAsTestFailure(); } diff --git a/testng-core/src/main/java/org/testng/CommandLineArgs.java b/testng-core/src/main/java/org/testng/CommandLineArgs.java index 32be0f3901..03866b2f29 100644 --- a/testng-core/src/main/java/org/testng/CommandLineArgs.java +++ b/testng-core/src/main/java/org/testng/CommandLineArgs.java @@ -251,4 +251,12 @@ public class CommandLineArgs { description = "Should TestNG report all iterations of a data driven test as individual skips, in-case of upstream failures.") public Boolean includeAllDataDrivenTestsWhenSkipping = false; + + public static final String PROPAGATE_DATA_PROVIDER_FAILURES_AS_TEST_FAILURE = + "-propagateDataProviderFailureAsTestFailure"; + + @Parameter( + names = PROPAGATE_DATA_PROVIDER_FAILURES_AS_TEST_FAILURE, + description = "Should TestNG consider failures in Data Providers as test failures.") + public Boolean propagateDataProviderFailureAsTestFailure = false; } diff --git a/testng-core/src/main/java/org/testng/TestNG.java b/testng-core/src/main/java/org/testng/TestNG.java index 4e538d8c5b..8ddbf093dd 100644 --- a/testng-core/src/main/java/org/testng/TestNG.java +++ b/testng-core/src/main/java/org/testng/TestNG.java @@ -15,6 +15,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.ServiceLoader; import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; @@ -596,6 +597,14 @@ public boolean getReportAllDataDrivenTestsAsSkipped() { return this.m_configuration.getReportAllDataDrivenTestsAsSkipped(); } + public void propagateDataProviderFailureAsTestFailure() { + this.m_configuration.propagateDataProviderFailureAsTestFailure(); + } + + public boolean isPropagateDataProviderFailureAsTestFailure() { + return this.m_configuration.isPropagateDataProviderFailureAsTestFailure(); + } + /** * Set the suites file names to be run by this TestNG object. This method tries to load and parse * the specified TestNG suite xml files. If a file is missing, it is ignored. @@ -1421,6 +1430,8 @@ public static TestNG privateMain(String[] argv, ITestListener listener) { * @param cla The command line parameters */ protected void configure(CommandLineArgs cla) { + Optional.ofNullable(cla.propagateDataProviderFailureAsTestFailure) + .ifPresent(value -> propagateDataProviderFailureAsTestFailure()); setReportAllDataDrivenTestsAsSkipped(cla.includeAllDataDrivenTestsWhenSkipping); if (cla.verbose != null) { setVerbose(cla.verbose); diff --git a/testng-core/src/main/java/org/testng/internal/Configuration.java b/testng-core/src/main/java/org/testng/internal/Configuration.java index c4a4c604f1..db11ea482a 100644 --- a/testng-core/src/main/java/org/testng/internal/Configuration.java +++ b/testng-core/src/main/java/org/testng/internal/Configuration.java @@ -35,6 +35,8 @@ public class Configuration implements IConfiguration { private boolean includeAllDataDrivenTestsWhenSkipping; + private boolean propagateDataProviderFailureAsTestFailure; + public Configuration() { init(new JDK15AnnotationFinder(new DefaultAnnotationTransformer())); } @@ -156,4 +158,14 @@ public void setReportAllDataDrivenTestsAsSkipped(boolean reportAllDataDrivenTest public boolean getReportAllDataDrivenTestsAsSkipped() { return this.includeAllDataDrivenTestsWhenSkipping; } + + @Override + public void propagateDataProviderFailureAsTestFailure() { + this.propagateDataProviderFailureAsTestFailure = true; + } + + @Override + public boolean isPropagateDataProviderFailureAsTestFailure() { + return propagateDataProviderFailureAsTestFailure; + } } diff --git a/testng-core/src/main/java/org/testng/internal/DataProviderMethod.java b/testng-core/src/main/java/org/testng/internal/DataProviderMethod.java index 263eff90eb..4ef2336a86 100644 --- a/testng-core/src/main/java/org/testng/internal/DataProviderMethod.java +++ b/testng-core/src/main/java/org/testng/internal/DataProviderMethod.java @@ -42,4 +42,9 @@ public boolean isParallel() { public List getIndices() { return annotation.getIndices(); } + + @Override + public boolean propagateFailureAsTestFailure() { + return annotation.isPropagateFailureAsTestFailure(); + } } diff --git a/testng-core/src/main/java/org/testng/internal/IConfiguration.java b/testng-core/src/main/java/org/testng/internal/IConfiguration.java index 9ab92d87d5..cd49902b16 100644 --- a/testng-core/src/main/java/org/testng/internal/IConfiguration.java +++ b/testng-core/src/main/java/org/testng/internal/IConfiguration.java @@ -55,4 +55,8 @@ default void setReportAllDataDrivenTestsAsSkipped(boolean reportAllDataDrivenTes default boolean getReportAllDataDrivenTestsAsSkipped() { return false; } + + void propagateDataProviderFailureAsTestFailure(); + + boolean isPropagateDataProviderFailureAsTestFailure(); } diff --git a/testng-core/src/main/java/org/testng/internal/annotations/DataProviderAnnotation.java b/testng-core/src/main/java/org/testng/internal/annotations/DataProviderAnnotation.java index 37c2e97fe0..12ca6e796d 100644 --- a/testng-core/src/main/java/org/testng/internal/annotations/DataProviderAnnotation.java +++ b/testng-core/src/main/java/org/testng/internal/annotations/DataProviderAnnotation.java @@ -9,6 +9,7 @@ public class DataProviderAnnotation extends BaseAnnotation implements IDataProvi private String m_name; private boolean m_parallel; private List m_indices; + private boolean m_bubbleUpFailures = false; @Override public boolean isParallel() { @@ -39,4 +40,14 @@ public List getIndices() { public void setIndices(List indices) { m_indices = indices; } + + @Override + public void propagateFailureAsTestFailure() { + m_bubbleUpFailures = true; + } + + @Override + public boolean isPropagateFailureAsTestFailure() { + return m_bubbleUpFailures; + } } diff --git a/testng-core/src/main/java/org/testng/internal/annotations/JDK15TagFactory.java b/testng-core/src/main/java/org/testng/internal/annotations/JDK15TagFactory.java index bdca20e6be..68feb7ddf0 100644 --- a/testng-core/src/main/java/org/testng/internal/annotations/JDK15TagFactory.java +++ b/testng-core/src/main/java/org/testng/internal/annotations/JDK15TagFactory.java @@ -424,6 +424,9 @@ private IAnnotation createDataProviderTag(Method method, Annotation a) { } result.setParallel(c.parallel()); result.setIndices(Ints.asList(c.indices())); + if (c.propagateFailureAsTestFailure()) { + result.propagateFailureAsTestFailure(); + } return result; } diff --git a/testng-core/src/main/java/org/testng/internal/invokers/ParameterHandler.java b/testng-core/src/main/java/org/testng/internal/invokers/ParameterHandler.java index f4c04be3f6..d159db2f06 100644 --- a/testng-core/src/main/java/org/testng/internal/invokers/ParameterHandler.java +++ b/testng-core/src/main/java/org/testng/internal/invokers/ParameterHandler.java @@ -3,7 +3,9 @@ import static org.testng.internal.Parameters.MethodParameters; import java.util.Map; +import java.util.Optional; import org.testng.DataProviderHolder; +import org.testng.IDataProviderMethod; import org.testng.ITestContext; import org.testng.ITestNGMethod; import org.testng.ITestObjectFactory; @@ -88,7 +90,12 @@ private ParameterBag handleParameters( } ITestResult result = TestResult.newTestResultWithCauseAs(testMethod, testContext, cause); - return new ParameterBag(result); + + boolean bubbleUpFailure = + Optional.ofNullable(testMethod.getDataProviderMethod()) + .map(IDataProviderMethod::propagateFailureAsTestFailure) + .orElse(false); + return new ParameterBag(result, bubbleUpFailure); } } @@ -99,15 +106,17 @@ private ParameterBag handleParameters( static class ParameterBag { final ParameterHolder parameterHolder; final ITestResult errorResult; + boolean bubbleUpFailures = false; ParameterBag(ParameterHolder parameterHolder) { this.parameterHolder = parameterHolder; this.errorResult = null; } - ParameterBag(ITestResult errorResult) { + ParameterBag(ITestResult errorResult, boolean bubbleUpFailures) { this.parameterHolder = null; this.errorResult = errorResult; + this.bubbleUpFailures = bubbleUpFailures; } boolean hasErrors() { @@ -119,5 +128,9 @@ boolean runInParallel() { && (parameterHolder.origin == ParameterHolder.ParameterOrigin.ORIGIN_DATA_PROVIDER && parameterHolder.dataProviderHolder.isParallel())); } + + boolean isBubbleUpFailures() { + return bubbleUpFailures; + } } } diff --git a/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java b/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java index f4a4ce4262..9b632cb1f4 100644 --- a/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java +++ b/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java @@ -924,7 +924,11 @@ public int invoke(int invCount) { if (bag.hasErrors()) { ITestResult tr = bag.errorResult; Throwable throwable = Objects.requireNonNull(tr).getThrowable(); - if (throwable instanceof TestNGException) { + boolean bubbleUpFailures = + m_configuration.isPropagateDataProviderFailureAsTestFailure() + || bag.isBubbleUpFailures(); + + if (throwable instanceof TestNGException || bubbleUpFailures) { tr.setStatus(ITestResult.FAILURE); m_notifier.addFailedTest(arguments.getTestMethod(), tr); } else { diff --git a/testng-core/src/test/java/test/dataprovider/DataProviderTest.java b/testng-core/src/test/java/test/dataprovider/DataProviderTest.java index 0827dc40d2..c19a11447c 100644 --- a/testng-core/src/test/java/test/dataprovider/DataProviderTest.java +++ b/testng-core/src/test/java/test/dataprovider/DataProviderTest.java @@ -7,13 +7,19 @@ import java.util.Collections; import java.util.List; import org.assertj.core.api.Condition; +import org.assertj.core.api.SoftAssertions; import org.testng.Assert; +import org.testng.IDataProviderMethod; +import org.testng.ITestNGMethod; import org.testng.ITestResult; import org.testng.TestListenerAdapter; import org.testng.TestNG; +import org.testng.TestNGException; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import org.testng.internal.reflect.MethodMatcherException; +import org.testng.xml.XmlClass; +import org.testng.xml.XmlTest; import test.InvokedMethodNameListener; import test.SimpleBaseTest; import test.dataprovider.issue1691.DataProviderDefinitionAtClassLevelAndNoTestMethodUsage; @@ -23,6 +29,12 @@ import test.dataprovider.issue1691.withinheritance.ChildClassHasFullDefinitionOfDataProviderAtClassLevel; import test.dataprovider.issue1691.withinheritance.ChildClassHasPartialDefinitionOfDataProviderAtClassLevel; import test.dataprovider.issue1691.withinheritance.ChildClassWithNoDataProviderInformationInTestMethod; +import test.dataprovider.issue1987.BaseClassSample; +import test.dataprovider.issue1987.DataProviderInBaseClassSample; +import test.dataprovider.issue1987.DataProviderInDifferentClassSample; +import test.dataprovider.issue1987.DataProviderInSameClassSample; +import test.dataprovider.issue1987.DataProviderTrackingListener; +import test.dataprovider.issue2504.SampleTestCaseListener; import test.dataprovider.issue2565.Data; import test.dataprovider.issue2565.SampleTestUsingConsumer; import test.dataprovider.issue2565.SampleTestUsingFunction; @@ -406,4 +418,97 @@ public void retryWithDataProvider() { assertThat(tla.getFailedTests()).size().isEqualTo(1); assertThat(tla.getSkippedTests()).size().isEqualTo(2); } + + @Test(description = "GITHUB-217", expectedExceptions = TestNGException.class) + public void ensureTestNGThrowsExceptionWhenAllTestsAreSkipped() { + TestNG testng = create(test.dataprovider.issue217.TestClassSample.class); + testng.toggleFailureIfAllTestsWereSkipped(true); + testng.run(); + } + + @Test(description = "GITHUB-217") + public void ensureTestNGFailsDueToDataProviderFailure() { + TestNG testng = create(test.dataprovider.issue217.TestClassSample.class); + testng.propagateDataProviderFailureAsTestFailure(); + testng.run(); + assertThat(testng.getStatus()).isEqualTo(1); + } + + @Test(description = "GITHUB-217") + public void ensureTestNGFailsDueToDataProviderFailure2() { + TestNG testng = create(test.dataprovider.issue217.AnotherTestClassSample.class); + testng.run(); + assertThat(testng.getStatus()).isEqualTo(1); + } + + @Test(description = "GITHUB-2255") + public void ensureDataProviderValuesAreVisibleToConfigMethods() { + TestNG testNG = create(test.dataprovider.issue2255.TestClassSample.class); + testNG.run(); + assertThat(test.dataprovider.issue2255.TestClassSample.data).containsExactly(100, 200); + } + + @Test(dataProvider = "testData", description = "GITHUB-1987") + public void extractDataProviderInfoWhenDpResidesInSameClass( + Class clazz, boolean performInstanceCheck, Class dataProviderClass) { + TestNG testng = create(clazz); + DataProviderTrackingListener listener = new DataProviderTrackingListener(); + testng.addListener(listener); + testng.run(); + ITestNGMethod method = listener.getResult().getMethod(); + IDataProviderMethod dpm = method.getDataProviderMethod(); + assertThat(dpm).isNotNull(); + if (performInstanceCheck) { + assertThat(dpm.getInstance()).isEqualTo(method.getInstance()); + } + assertThat(dpm.getMethod().getName()).isEqualTo("getData"); + assertThat(dpm.getInstance().getClass()).isEqualTo(dataProviderClass); + } + + @DataProvider(name = "testData") + public Object[][] getTestData() { + return new Object[][] { + {DataProviderInSameClassSample.class, true, DataProviderInSameClassSample.class}, + {DataProviderInBaseClassSample.class, true, DataProviderInBaseClassSample.class}, + {DataProviderInDifferentClassSample.class, false, BaseClassSample.class} + }; + } + + @Test(description = "GITHUB-2267") + public void ensureDynamicRetryAnalyzersAreHonouredForDataDrivenTest() { + TestNG testng = create(test.dataprovider.issue2267.TestClassSample.class); + TestListenerAdapter tla = new TestListenerAdapter(); + testng.addListener(tla); + testng.run(); + assertThat(tla.getFailedTests()).size().isEqualTo(1); + assertThat(tla.getSkippedTests()).size().isEqualTo(1); + } + + @Test(description = "GITHUB-2327") + public void ensureDataProviderParametersAreAlwaysAvailableForListeners() { + TestNG testng = create(test.dataprovider.issue2327.TestClassSample.class); + TestListenerAdapter tla = new TestListenerAdapter(); + testng.addListener(tla); + testng.run(); + + assertThat(tla.getSkippedTests().size()).isEqualTo(2); + SoftAssertions assertions = new SoftAssertions(); + + for (ITestResult skippedTest : tla.getSkippedTests()) { + assertions.assertThat(skippedTest.getParameters()).isNotEmpty(); + } + assertions.assertAll(); + } + + @Test(description = "GITHUB-2504") + public void ensureParametersCopiedOnConfigFailures() { + XmlTest xmltest = createXmlTest("2504_suite", "2504_test"); + xmltest.setXmlClasses( + Collections.singletonList(new XmlClass(test.dataprovider.issue2504.TestClassSample.class))); + TestNG testNG = create(Collections.singletonList(xmltest.getSuite())); + SampleTestCaseListener listener = new SampleTestCaseListener(); + testNG.addListener(listener); + testNG.run(); + assertThat(listener.getParameters()).containsExactlyElementsOf(Arrays.asList(1, 2, 3, 4, 5)); + } } diff --git a/testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInBaseClass.java b/testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInBaseClassSample.java similarity index 67% rename from testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInBaseClass.java rename to testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInBaseClassSample.java index f86448cfff..e448dba051 100644 --- a/testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInBaseClass.java +++ b/testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInBaseClassSample.java @@ -2,7 +2,7 @@ import org.testng.annotations.Test; -public class DataProviderInBaseClass extends BaseClassSample { +public class DataProviderInBaseClassSample extends BaseClassSample { @Test(dataProvider = "dp") public void testMethod(int i) {} diff --git a/testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInDifferentClass.java b/testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInDifferentClassSample.java similarity index 78% rename from testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInDifferentClass.java rename to testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInDifferentClassSample.java index d4ead013a7..48b35c627b 100644 --- a/testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInDifferentClass.java +++ b/testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInDifferentClassSample.java @@ -2,7 +2,7 @@ import org.testng.annotations.Test; -public class DataProviderInDifferentClass { +public class DataProviderInDifferentClassSample { @Test(dataProvider = "dp", dataProviderClass = BaseClassSample.class) public void testMethod(int i) {} diff --git a/testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInSameClass.java b/testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInSameClassSample.java similarity index 86% rename from testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInSameClass.java rename to testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInSameClassSample.java index 493ef2fa5b..a594126352 100644 --- a/testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInSameClass.java +++ b/testng-core/src/test/java/test/dataprovider/issue1987/DataProviderInSameClassSample.java @@ -3,7 +3,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class DataProviderInSameClass { +public class DataProviderInSameClassSample { @Test(dataProvider = "dp") public void testMethod(int i) {} diff --git a/testng-core/src/test/java/test/dataprovider/issue1987/IssueTest.java b/testng-core/src/test/java/test/dataprovider/issue1987/IssueTest.java deleted file mode 100644 index 4bc0f2978b..0000000000 --- a/testng-core/src/test/java/test/dataprovider/issue1987/IssueTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package test.dataprovider.issue1987; - -import static org.assertj.core.api.Assertions.assertThat; - -import org.testng.IDataProviderMethod; -import org.testng.ITestNGMethod; -import org.testng.TestNG; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; -import test.SimpleBaseTest; - -public class IssueTest extends SimpleBaseTest { - - @Test(dataProvider = "testData", description = "GITHUB-1987") - public void extractDataProviderInfoWhenDpResidesInSameClass( - Class clazz, boolean performInstanceCheck, Class dataProviderClass) { - TestNG testng = create(clazz); - DataProviderTrackingListener listener = new DataProviderTrackingListener(); - testng.addListener(listener); - testng.run(); - ITestNGMethod method = listener.getResult().getMethod(); - IDataProviderMethod dpm = method.getDataProviderMethod(); - assertThat(dpm).isNotNull(); - if (performInstanceCheck) { - assertThat(dpm.getInstance()).isEqualTo(method.getInstance()); - } - assertThat(dpm.getMethod().getName()).isEqualTo("getData"); - assertThat(dpm.getInstance().getClass()).isEqualTo(dataProviderClass); - } - - @DataProvider(name = "testData") - public Object[][] getTestData() { - return new Object[][] { - {DataProviderInSameClass.class, true, DataProviderInSameClass.class}, - {DataProviderInBaseClass.class, true, DataProviderInBaseClass.class}, - {DataProviderInDifferentClass.class, false, BaseClassSample.class} - }; - } -} diff --git a/testng-core/src/test/java/test/dataprovider/issue217/AnotherTestClassSample.java b/testng-core/src/test/java/test/dataprovider/issue217/AnotherTestClassSample.java new file mode 100644 index 0000000000..10f3e0c3dc --- /dev/null +++ b/testng-core/src/test/java/test/dataprovider/issue217/AnotherTestClassSample.java @@ -0,0 +1,16 @@ +package test.dataprovider.issue217; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class AnotherTestClassSample { + public static final String DP_NAME = "dataProvider"; + + @DataProvider(name = DP_NAME, propagateFailureAsTestFailure = true) + public static Object[][] getData() { + throw new IllegalStateException("guess me!"); + } + + @Test(dataProvider = DP_NAME) + public void test() {} +} diff --git a/testng-core/src/test/java/test/dataprovider/issue217/IssueTest.java b/testng-core/src/test/java/test/dataprovider/issue217/IssueTest.java deleted file mode 100644 index 3d103a916c..0000000000 --- a/testng-core/src/test/java/test/dataprovider/issue217/IssueTest.java +++ /dev/null @@ -1,16 +0,0 @@ -package test.dataprovider.issue217; - -import org.testng.TestNG; -import org.testng.TestNGException; -import org.testng.annotations.Test; -import test.SimpleBaseTest; - -public class IssueTest extends SimpleBaseTest { - - @Test(description = "GITHUB-217", expectedExceptions = TestNGException.class) - public void ensureTestNGThrowsExceptionWhenAllTestsAreSkipped() { - TestNG testng = create(SampleTestCase.class); - testng.toggleFailureIfAllTestsWereSkipped(true); - testng.run(); - } -} diff --git a/testng-core/src/test/java/test/dataprovider/issue217/SampleTestCase.java b/testng-core/src/test/java/test/dataprovider/issue217/TestClassSample.java similarity index 92% rename from testng-core/src/test/java/test/dataprovider/issue217/SampleTestCase.java rename to testng-core/src/test/java/test/dataprovider/issue217/TestClassSample.java index 4a03063426..cf9a748679 100644 --- a/testng-core/src/test/java/test/dataprovider/issue217/SampleTestCase.java +++ b/testng-core/src/test/java/test/dataprovider/issue217/TestClassSample.java @@ -3,7 +3,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class SampleTestCase { +public class TestClassSample { public static final String DP_NAME = "dataProvider"; @DataProvider(name = DP_NAME) diff --git a/testng-core/src/test/java/test/dataprovider/issue2255/IssueTest.java b/testng-core/src/test/java/test/dataprovider/issue2255/IssueTest.java deleted file mode 100644 index 204d61ea12..0000000000 --- a/testng-core/src/test/java/test/dataprovider/issue2255/IssueTest.java +++ /dev/null @@ -1,17 +0,0 @@ -package test.dataprovider.issue2255; - -import static org.assertj.core.api.Assertions.assertThat; - -import org.testng.TestNG; -import org.testng.annotations.Test; -import test.SimpleBaseTest; - -public class IssueTest extends SimpleBaseTest { - - @Test(description = "GITHUB-2255") - public void runTest() { - TestNG testNG = create(SampleTestCase.class); - testNG.run(); - assertThat(SampleTestCase.data).containsExactly(100, 200); - } -} diff --git a/testng-core/src/test/java/test/dataprovider/issue2255/SampleTestCase.java b/testng-core/src/test/java/test/dataprovider/issue2255/TestClassSample.java similarity index 90% rename from testng-core/src/test/java/test/dataprovider/issue2255/SampleTestCase.java rename to testng-core/src/test/java/test/dataprovider/issue2255/TestClassSample.java index 1905268fc5..a9edc6d474 100644 --- a/testng-core/src/test/java/test/dataprovider/issue2255/SampleTestCase.java +++ b/testng-core/src/test/java/test/dataprovider/issue2255/TestClassSample.java @@ -12,9 +12,9 @@ import org.testng.annotations.Test; import test.SimpleBaseTest; -public class SampleTestCase extends SimpleBaseTest { +public class TestClassSample extends SimpleBaseTest { - static final List data = new ArrayList<>(); + public static final List data = new ArrayList<>(); private static final AtomicInteger counter = new AtomicInteger(0); @BeforeMethod diff --git a/testng-core/src/test/java/test/dataprovider/issue2267/IssueTest.java b/testng-core/src/test/java/test/dataprovider/issue2267/IssueTest.java deleted file mode 100644 index 0fe72c53b1..0000000000 --- a/testng-core/src/test/java/test/dataprovider/issue2267/IssueTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package test.dataprovider.issue2267; - -import static org.assertj.core.api.Assertions.assertThat; - -import org.testng.TestListenerAdapter; -import org.testng.TestNG; -import org.testng.annotations.Test; -import test.SimpleBaseTest; - -public class IssueTest extends SimpleBaseTest { - - @Test(description = "GITHUB-2267") - public void runTest() { - TestNG testng = create(SampleTestCase.class); - TestListenerAdapter tla = new TestListenerAdapter(); - testng.addListener(tla); - testng.run(); - assertThat(tla.getFailedTests()).size().isEqualTo(1); - assertThat(tla.getSkippedTests()).size().isEqualTo(1); - } -} diff --git a/testng-core/src/test/java/test/dataprovider/issue2267/SampleTestCase.java b/testng-core/src/test/java/test/dataprovider/issue2267/TestClassSample.java similarity index 90% rename from testng-core/src/test/java/test/dataprovider/issue2267/SampleTestCase.java rename to testng-core/src/test/java/test/dataprovider/issue2267/TestClassSample.java index 78f3b11218..c1f03117c3 100644 --- a/testng-core/src/test/java/test/dataprovider/issue2267/SampleTestCase.java +++ b/testng-core/src/test/java/test/dataprovider/issue2267/TestClassSample.java @@ -7,7 +7,7 @@ import test.SimpleBaseTest; @Listeners(CustomListener.class) -public class SampleTestCase extends SimpleBaseTest { +public class TestClassSample extends SimpleBaseTest { @DataProvider(name = "dp") public Object[][] dp() { diff --git a/testng-core/src/test/java/test/dataprovider/issue2327/IssueTest.java b/testng-core/src/test/java/test/dataprovider/issue2327/IssueTest.java deleted file mode 100644 index 7346fb7a7f..0000000000 --- a/testng-core/src/test/java/test/dataprovider/issue2327/IssueTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package test.dataprovider.issue2327; - -import static org.assertj.core.api.Assertions.assertThat; - -import org.testng.ITestResult; -import org.testng.TestListenerAdapter; -import org.testng.TestNG; -import org.testng.annotations.Test; -import test.SimpleBaseTest; - -public class IssueTest extends SimpleBaseTest { - - @Test(description = "GITHUB-2327") - public void runTest() { - TestNG testng = create(SampleTestCase.class); - TestListenerAdapter tla = new TestListenerAdapter(); - testng.addListener(tla); - testng.run(); - - assertThat(tla.getSkippedTests().size()).isEqualTo(2); - - for (ITestResult skippedTest : tla.getSkippedTests()) { - assertThat(skippedTest.getParameters()).isNotEmpty(); - } - } -} diff --git a/testng-core/src/test/java/test/dataprovider/issue2327/SampleTestCase.java b/testng-core/src/test/java/test/dataprovider/issue2327/TestClassSample.java similarity index 95% rename from testng-core/src/test/java/test/dataprovider/issue2327/SampleTestCase.java rename to testng-core/src/test/java/test/dataprovider/issue2327/TestClassSample.java index b42ca11a60..bc4d643bb0 100644 --- a/testng-core/src/test/java/test/dataprovider/issue2327/SampleTestCase.java +++ b/testng-core/src/test/java/test/dataprovider/issue2327/TestClassSample.java @@ -7,7 +7,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class SampleTestCase { +public class TestClassSample { @BeforeMethod(alwaysRun = true) public void setup(Method method, Object[] params) { diff --git a/testng-core/src/test/java/test/dataprovider/issue2504/IssueTest.java b/testng-core/src/test/java/test/dataprovider/issue2504/IssueTest.java deleted file mode 100644 index f5ccb1a4bb..0000000000 --- a/testng-core/src/test/java/test/dataprovider/issue2504/IssueTest.java +++ /dev/null @@ -1,25 +0,0 @@ -package test.dataprovider.issue2504; - -import java.util.Arrays; -import java.util.Collections; -import org.assertj.core.api.Assertions; -import org.testng.TestNG; -import org.testng.annotations.Test; -import org.testng.xml.XmlClass; -import org.testng.xml.XmlTest; -import test.SimpleBaseTest; - -public class IssueTest extends SimpleBaseTest { - - @Test(description = "GITHUB-2504") - public void ensureParametersCopiedOnConfigFailures() { - XmlTest xmltest = createXmlTest("2504_suite", "2504_test"); - xmltest.setXmlClasses(Collections.singletonList(new XmlClass(SampleTestCase.class))); - TestNG testNG = create(Collections.singletonList(xmltest.getSuite())); - SampleTestCaseListener listener = new SampleTestCaseListener(); - testNG.addListener(listener); - testNG.run(); - Assertions.assertThat(listener.getParameters()) - .containsExactlyElementsOf(Arrays.asList(1, 2, 3, 4, 5)); - } -} diff --git a/testng-core/src/test/java/test/dataprovider/issue2504/SampleTestCase.java b/testng-core/src/test/java/test/dataprovider/issue2504/TestClassSample.java similarity index 94% rename from testng-core/src/test/java/test/dataprovider/issue2504/SampleTestCase.java rename to testng-core/src/test/java/test/dataprovider/issue2504/TestClassSample.java index f3ddf1d943..1cb5a5e1a2 100644 --- a/testng-core/src/test/java/test/dataprovider/issue2504/SampleTestCase.java +++ b/testng-core/src/test/java/test/dataprovider/issue2504/TestClassSample.java @@ -5,7 +5,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class SampleTestCase { +public class TestClassSample { @DataProvider public Object[][] data() { diff --git a/testng-core/src/test/resources/testng.xml b/testng-core/src/test/resources/testng.xml index 1a7644b710..5ac8b0caa1 100644 --- a/testng-core/src/test/resources/testng.xml +++ b/testng-core/src/test/resources/testng.xml @@ -600,13 +600,7 @@ - - - - - -