From d6450021108a18358688e38673a929c68c7dbb98 Mon Sep 17 00:00:00 2001 From: Krishnan Mahadevan Date: Fri, 4 Nov 2022 19:09:57 +0530 Subject: [PATCH] Add config key for callback discrepancy behavior Closes #2818 --- CHANGES.txt | 1 + .../org/testng/internal/RuntimeBehavior.java | 5 +++ .../internal/invokers/ConfigInvoker.java | 6 +++- .../testng/internal/invokers/TestInvoker.java | 6 +++- .../src/test/java/test/hook/HookableTest.java | 24 ++++++++++++++ .../test/hook/samples/CallBackSample.java | 32 +++++++++++++++++++ 6 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 testng-core/src/test/java/test/hook/samples/CallBackSample.java diff --git a/CHANGES.txt b/CHANGES.txt index a8ff79ac8f..ed88ccd4ef 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ Current +Fixed: GITHUB2818: Add configuration key for callback discrepancy behavior (Krishnan Mahadevan) Fixed: GITHUB-2819: Ability to retry a data provider in case of failures (Krishnan Mahadevan) Fixed: GITHUB-2308: StringIndexOutOfBoundsException in findClassesInPackage - Surefire/Maven - JDK 11 fails (Krishnan Mahadevan) Fixed: GITHUB:2788: TestResult.isSuccess() is TRUE when test fails due to expectedExceptions (Krishnan Mahadevan) diff --git a/testng-core-api/src/main/java/org/testng/internal/RuntimeBehavior.java b/testng-core-api/src/main/java/org/testng/internal/RuntimeBehavior.java index 4be13a434a..34d9996e50 100644 --- a/testng-core-api/src/main/java/org/testng/internal/RuntimeBehavior.java +++ b/testng-core-api/src/main/java/org/testng/internal/RuntimeBehavior.java @@ -14,9 +14,14 @@ public final class RuntimeBehavior { private static final String MEMORY_FRIENDLY_MODE = "testng.memory.friendly"; public static final String STRICTLY_HONOUR_PARALLEL_MODE = "testng.strict.parallel"; public static final String TESTNG_DEFAULT_VERBOSE = "testng.default.verbose"; + public static final String IGNORE_CALLBACK_INVOCATION_SKIPS = "testng.ignore.callback.skip"; private RuntimeBehavior() {} + public static boolean ignoreCallbackInvocationSkips() { + return Boolean.getBoolean(IGNORE_CALLBACK_INVOCATION_SKIPS); + } + public static boolean strictParallelism() { return Boolean.getBoolean(STRICTLY_HONOUR_PARALLEL_MODE); } diff --git a/testng-core/src/main/java/org/testng/internal/invokers/ConfigInvoker.java b/testng-core/src/main/java/org/testng/internal/invokers/ConfigInvoker.java index c5c102a5ce..9ecb0a4a45 100644 --- a/testng-core/src/main/java/org/testng/internal/invokers/ConfigInvoker.java +++ b/testng-core/src/main/java/org/testng/internal/invokers/ConfigInvoker.java @@ -362,7 +362,11 @@ private void invokeConfigurationMethod( tm, method, targetInstance, params, testResult); } boolean testStatusRemainedUnchanged = testResult.isNotRunning(); - if (usesConfigurableInstance && willfullyIgnored && testStatusRemainedUnchanged) { + boolean throwException = !RuntimeBehavior.ignoreCallbackInvocationSkips(); + if (throwException + && usesConfigurableInstance + && willfullyIgnored + && testStatusRemainedUnchanged) { throw new ConfigurationNotInvokedException(tm); } testResult.setStatus(ITestResult.SUCCESS); diff --git a/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java b/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java index 7f16ba92d1..03e1e2e3f1 100644 --- a/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java +++ b/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java @@ -691,7 +691,11 @@ private ITestResult invokeMethod( hookableInstance); } boolean testStatusRemainedUnchanged = testResult.isNotRunning(); - if (usesHookableInstance && willfullyIgnored && testStatusRemainedUnchanged) { + boolean throwException = !RuntimeBehavior.ignoreCallbackInvocationSkips(); + if (throwException + && usesHookableInstance + && willfullyIgnored + && testStatusRemainedUnchanged) { TestNotInvokedException tn = new TestNotInvokedException(arguments.tm); testResult.setThrowable(tn); setTestStatus(testResult, ITestResult.FAILURE); diff --git a/testng-core/src/test/java/test/hook/HookableTest.java b/testng-core/src/test/java/test/hook/HookableTest.java index f46b23cf2f..65399081e7 100644 --- a/testng-core/src/test/java/test/hook/HookableTest.java +++ b/testng-core/src/test/java/test/hook/HookableTest.java @@ -16,7 +16,9 @@ import org.testng.TestNG; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import org.testng.internal.RuntimeBehavior; import test.SimpleBaseTest; +import test.hook.samples.CallBackSample; import test.hook.samples.ConfigurableFailureSample; import test.hook.samples.ConfigurableFailureWithStatusAlteredSample; import test.hook.samples.ConfigurableSuccessSample; @@ -196,6 +198,28 @@ public void configurableFailureWithStatusAltered() { assertions.assertAll(); } + @Test(description = "GITHUB-2818", dataProvider = "callback") + public void tesFailuresFromCallbackInvocationSkipsCanBeDisabledViaJVMArgs( + Class cls, int expected) { + try { + System.setProperty(RuntimeBehavior.IGNORE_CALLBACK_INVOCATION_SKIPS, "true"); + TestNG testng = create(cls); + testng.run(); + int status = testng.getStatus(); + assertThat(status).isEqualTo(expected); + } finally { + System.setProperty(RuntimeBehavior.IGNORE_CALLBACK_INVOCATION_SKIPS, ""); + } + } + + @DataProvider(name = "callback") + public Object[][] getCallBackClasses() { + return new Object[][] { + {CallBackSample.ConfigCallBackSkipTestCase.class, 0}, + {CallBackSample.TestCallBackSkipTestCase.class, 1}, + }; + } + public static class TestResultsCollector implements IInvokedMethodListener { private final List passed = new ArrayList<>(); private final List invoked = new ArrayList<>(); diff --git a/testng-core/src/test/java/test/hook/samples/CallBackSample.java b/testng-core/src/test/java/test/hook/samples/CallBackSample.java new file mode 100644 index 0000000000..6bb92c29ed --- /dev/null +++ b/testng-core/src/test/java/test/hook/samples/CallBackSample.java @@ -0,0 +1,32 @@ +package test.hook.samples; + +import org.testng.IConfigurable; +import org.testng.IConfigureCallBack; +import org.testng.IHookCallBack; +import org.testng.IHookable; +import org.testng.ITestResult; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +public class CallBackSample { + + public static class ConfigCallBackSkipTestCase implements IConfigurable { + + @Override + public void run(IConfigureCallBack callBack, ITestResult testResult) {} + + @BeforeMethod + public void beforeMethod() {} + + @Test + public void testMethod() {} + } + + public static class TestCallBackSkipTestCase implements IHookable { + @Override + public void run(IHookCallBack callBack, ITestResult testResult) {} + + @Test + public void testMethod() {} + } +}