-
Notifications
You must be signed in to change notification settings - Fork 283
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement flaky tests retry for Spock
- Loading branch information
1 parent
30789d4
commit bdf2753
Showing
24 changed files
with
1,851 additions
and
67 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
66 changes: 66 additions & 0 deletions
66
...adog/trace/instrumentation/junit5/retry/JUnit5SpockParameterizedRetryInstrumentation.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| package datadog.trace.instrumentation.junit5.retry; | ||
|
|
||
| import static net.bytebuddy.matcher.ElementMatchers.isConstructor; | ||
|
|
||
| import com.google.auto.service.AutoService; | ||
| import datadog.trace.agent.tooling.Instrumenter; | ||
| import datadog.trace.api.Config; | ||
| import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; | ||
| import java.util.Map; | ||
| import java.util.Set; | ||
| import java.util.concurrent.CompletableFuture; | ||
| import net.bytebuddy.asm.Advice; | ||
| import org.junit.platform.engine.EngineExecutionListener; | ||
| import org.junit.platform.engine.TestDescriptor; | ||
|
|
||
| /** | ||
| * Applies a patch to Spock's parameterized tests executor, needed to support retries for | ||
| * parameterized tests | ||
| */ | ||
| @AutoService(Instrumenter.class) | ||
| public class JUnit5SpockParameterizedRetryInstrumentation extends Instrumenter.CiVisibility | ||
| implements Instrumenter.ForSingleType { | ||
|
|
||
| public JUnit5SpockParameterizedRetryInstrumentation() { | ||
| super("ci-visibility", "junit-5", "junit-5-spock", "test-retry"); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isApplicable(Set<TargetSystem> enabledSystems) { | ||
| return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityFlakyRetryEnabled(); | ||
| } | ||
|
|
||
| @Override | ||
| public String instrumentedType() { | ||
| return "org.spockframework.runtime.ParameterizedFeatureChildExecutor"; | ||
| } | ||
|
|
||
| @Override | ||
| public String[] helperClassNames() { | ||
| return new String[] { | ||
| packageName + ".SpockParameterizedExecutionListener", | ||
| }; | ||
| } | ||
|
|
||
| @Override | ||
| public void adviceTransformations(AdviceTransformation transformation) { | ||
| transformation.applyAdvice( | ||
| isConstructor(), | ||
| JUnit5SpockParameterizedRetryInstrumentation.class.getName() | ||
| + "$SpockParameterizedRetryAdvice"); | ||
| } | ||
|
|
||
| public static class SpockParameterizedRetryAdvice { | ||
|
|
||
| @SuppressFBWarnings( | ||
| value = "UC_USELESS_OBJECT", | ||
| justification = "executionListener is a field in the instrumented class") | ||
| @Advice.OnMethodExit | ||
| public static void afterConstructor( | ||
| @Advice.FieldValue(value = "executionListener", readOnly = false) | ||
| EngineExecutionListener executionListener, | ||
| @Advice.FieldValue("pending") Map<TestDescriptor, CompletableFuture<?>> pending) { | ||
| executionListener = new SpockParameterizedExecutionListener(executionListener, pending); | ||
| } | ||
| } | ||
| } |
50 changes: 50 additions & 0 deletions
50
.../java/datadog/trace/instrumentation/junit5/retry/SpockParameterizedExecutionListener.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| package datadog.trace.instrumentation.junit5.retry; | ||
|
|
||
| import java.util.Map; | ||
| import java.util.concurrent.CompletableFuture; | ||
| import org.junit.platform.engine.EngineExecutionListener; | ||
| import org.junit.platform.engine.TestDescriptor; | ||
| import org.junit.platform.engine.TestExecutionResult; | ||
| import org.junit.platform.engine.reporting.ReportEntry; | ||
|
|
||
| public class SpockParameterizedExecutionListener implements EngineExecutionListener { | ||
| private final EngineExecutionListener delegate; | ||
| private final Map<TestDescriptor, CompletableFuture<?>> pending; | ||
|
|
||
| public SpockParameterizedExecutionListener( | ||
| EngineExecutionListener delegate, Map<TestDescriptor, CompletableFuture<?>> pending) { | ||
| this.delegate = delegate; | ||
| this.pending = pending; | ||
| } | ||
|
|
||
| @Override | ||
| public void dynamicTestRegistered(TestDescriptor testDescriptor) { | ||
| delegate.dynamicTestRegistered(testDescriptor); | ||
| if (RetryContext.RETRY_ATTEMPT_TEST_ID_SEGMENT.equals( | ||
| testDescriptor.getUniqueId().getLastSegment().getType())) { | ||
| // register generated retry descriptor | ||
| pending.put(testDescriptor, new CompletableFuture<>()); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void executionSkipped(TestDescriptor testDescriptor, String reason) { | ||
| delegate.executionSkipped(testDescriptor, reason); | ||
| } | ||
|
|
||
| @Override | ||
| public void executionStarted(TestDescriptor testDescriptor) { | ||
| delegate.executionStarted(testDescriptor); | ||
| } | ||
|
|
||
| @Override | ||
| public void executionFinished( | ||
| TestDescriptor testDescriptor, TestExecutionResult testExecutionResult) { | ||
| delegate.executionFinished(testDescriptor, testExecutionResult); | ||
| } | ||
|
|
||
| @Override | ||
| public void reportingEntryPublished(TestDescriptor testDescriptor, ReportEntry entry) { | ||
| delegate.reportingEntryPublished(testDescriptor, entry); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
16 changes: 16 additions & 0 deletions
16
...n/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestFailedParameterizedSpock.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| package org.example | ||
|
|
||
| import spock.lang.Specification | ||
|
|
||
| class TestFailedParameterizedSpock extends Specification { | ||
|
|
||
| def "test add #a and #b"() { | ||
| expect: | ||
| a + b == c | ||
|
|
||
| where: | ||
| a | b | c | ||
| 1 | 2 | 3 | ||
| 4 | 4 | 44 | ||
| } | ||
| } |
11 changes: 11 additions & 0 deletions
11
...nstrumentation/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestFailedSpock.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| package org.example | ||
|
|
||
| import spock.lang.Specification | ||
|
|
||
| class TestFailedSpock extends Specification { | ||
|
|
||
| def "test failed"() { | ||
| expect: | ||
| 1 == 2 | ||
| } | ||
| } |
18 changes: 18 additions & 0 deletions
18
.../spock-junit-5/src/test/groovy/org/example/TestFailedThenSucceedParameterizedSpock.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| package org.example | ||
|
|
||
| import spock.lang.Specification | ||
|
|
||
| class TestFailedThenSucceedParameterizedSpock extends Specification { | ||
|
|
||
| public static int testExecutionCount = 0 | ||
|
|
||
| def "test add #a and #b"() { | ||
| expect: | ||
| c == 3 && ++testExecutionCount > 2 | ||
|
|
||
| where: | ||
| a | b | c | ||
| 1 | 2 | 3 | ||
| 4 | 4 | 8 | ||
| } | ||
| } |
13 changes: 13 additions & 0 deletions
13
...ion/junit-5.3/spock-junit-5/src/test/groovy/org/example/TestFailedThenSucceedSpock.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| package org.example | ||
|
|
||
| import spock.lang.Specification | ||
|
|
||
| class TestFailedThenSucceedSpock extends Specification { | ||
|
|
||
| public static int testExecutionCount = 0 | ||
|
|
||
| def "test failed then succeed"() { | ||
| expect: | ||
| ++testExecutionCount > 3 | ||
| } | ||
| } |
1 change: 1 addition & 0 deletions
1
...ntation/junit-5.3/spock-junit-5/src/test/resources/test-failed-then-succeed/coverages.ftl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| [ ] |
Oops, something went wrong.