diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/LocalFilesBasedConfig.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/LocalFilesBasedConfig.java index b80d67b30..6be47bc03 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/LocalFilesBasedConfig.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/LocalFilesBasedConfig.java @@ -114,6 +114,20 @@ protected TcBotJsonConfig getConfig() { return Strings.isNullOrEmpty(srvCode) ? ITcBotConfig.DEFAULT_SERVER_CODE : srvCode; } + /** {@inheritDoc} */ + @Override public Integer flakyRate() { + Integer flakyRate = getConfig().flakyRate(); + + return flakyRate == null || flakyRate < 0 || flakyRate > 100 ? ITcBotConfig.DEFAULT_FLAKY_RATE : flakyRate; + } + + /** {@inheritDoc} */ + @Override public Double confidence() { + Double confidence = getConfig().confidence(); + + return confidence == null || confidence < 0 || confidence > 1 ? ITcBotConfig.DEFAULT_CONFIDENCE : confidence; + } + @Override public ITrackedBranchesConfig getTrackedBranches() { return getConfig(); diff --git a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java index e96a0bedf..fe1ef2bc3 100644 --- a/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java +++ b/ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java @@ -30,11 +30,13 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.inject.Inject; import javax.inject.Provider; import org.apache.ignite.ci.issue.Issue; import org.apache.ignite.ci.issue.IssueKey; +import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation; import org.apache.ignite.tcbot.engine.issue.IIssuesStorage; import org.apache.ignite.tcbot.engine.issue.IssueType; import org.apache.ignite.ci.jobs.CheckQueueJob; @@ -64,11 +66,14 @@ import org.apache.ignite.tcignited.ITeamcityIgnitedProvider; import org.apache.ignite.tcignited.SyncMode; import org.apache.ignite.tcignited.history.IRunHistory; +import org.apache.ignite.tcignited.history.InvocationData; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.ignite.tcignited.buildref.BranchEquivalence.normalizeBranch; +import static org.apache.ignite.tcignited.history.RunStatus.RES_FAILURE; +import static org.apache.ignite.tcignited.history.RunStatus.RES_OK; /** * @@ -471,15 +476,49 @@ private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited, type = IssueType.newFailure; final String flakyComments = runStat.getFlakyComments(); - if (!Strings.isNullOrEmpty(flakyComments)) { - if (runStat.detectTemplate(EventTemplates.newFailureForFlakyTest) == null) { - logger.info("Skipping registering new issue for test fail:" + - " Test seems to be flaky " + name + ": " + flakyComments); + if (!Strings.isNullOrEmpty(flakyComments) && + runStat.detectTemplate(EventTemplates.newFailureForFlakyTest) != null) + type = IssueType.newFailureForFlakyTest; + } + } - firstFailedBuildId = null; - } + double flakyRate = 0; + + if (firstFailedBuildId == null || type == null) { + List invocations = runStat.getInvocations(). + filter(invocation -> invocation != null && invocation.status() != InvocationData.MISSING) + .collect(Collectors.toList()); + + int confidenceOkTestsRow = Math.max(1, + (int) Math.ceil(Math.log(1 - cfg.confidence()) / Math.log(1 - cfg.flakyRate() / 100.0))); + + if (invocations.size() >= confidenceOkTestsRow * 2) { + List lastInvocations = + invocations.subList(invocations.size() - confidenceOkTestsRow * 2, invocations.size()); + + int stableTestRuns = 0; + + for (int i = 0; i < confidenceOkTestsRow; i++) { + if (lastInvocations.get(i).status() == RES_OK.getCode()) + stableTestRuns++; else - type = IssueType.newFailureForFlakyTest; + break; + } + + if (stableTestRuns == confidenceOkTestsRow) { + long failedTestRuns = 0; + + for (int i = confidenceOkTestsRow; i < confidenceOkTestsRow * 2; i++) { + if (lastInvocations.get(i).status() == RES_FAILURE.getCode()) + failedTestRuns++; + } + + flakyRate = (double) failedTestRuns / confidenceOkTestsRow * 100; + + if (flakyRate > cfg.flakyRate()) { + type = IssueType.newTestWithHighFlakyRate; + firstFailedBuildId = lastInvocations.get(confidenceOkTestsRow).buildId(); + } } } } @@ -501,6 +540,7 @@ private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited, issue.trackedBranchName = trackedBranch; issue.displayName = testFailure.testName; issue.webUrl = testFailure.webUrl; + issue.flakyRate = flakyRate; issue.buildTags.addAll(suiteTags); diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java index a8ae186e2..f6c1f7bcd 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java @@ -98,6 +98,14 @@ public MockBasedTcBotModule() { return ITcBotConfig.DEFAULT_SERVER_CODE; } + @Override public Integer flakyRate() { + return DEFAULT_FLAKY_RATE; + } + + @Override public Double confidence() { + return DEFAULT_CONFIDENCE; + } + @Override public ITrackedBranchesConfig getTrackedBranches() { return tracked; } diff --git a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/issue/IssueDetectorTest.java b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/issue/IssueDetectorTest.java index bc5e4c917..aeb07fcae 100644 --- a/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/issue/IssueDetectorTest.java +++ b/ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/issue/IssueDetectorTest.java @@ -106,8 +106,8 @@ public void testDetector() { Map buildWoChanges = new TreeMap() { { - put("testFailedShoudlBeConsideredAsFlaky", "0000011111"); - put("testFlakyStableFailure", "0000011111111111"); + put("testFailedShouldBeConsideredAsFlaky", "0000011111"); + put("testFlakyStableFailure", "0000010101100101"); } }; diff --git a/tcbot-engine/src/main/java/org/apache/ignite/ci/issue/Issue.java b/tcbot-engine/src/main/java/org/apache/ignite/ci/issue/Issue.java index 692312252..d7651306e 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/ci/issue/Issue.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/ci/issue/Issue.java @@ -47,6 +47,8 @@ public class Issue { /** Display type. for issue. Kept for backward compatibilty with older records without type code. */ private String displayType; + public double flakyRate; + @Nullable public String trackedBranchName; diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/board/BoardService.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/board/BoardService.java index 4bfbba6e9..9042e1e73 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/board/BoardService.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/board/BoardService.java @@ -19,6 +19,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -43,6 +44,7 @@ import org.apache.ignite.tcbot.common.util.FutureUtil; import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor; import org.apache.ignite.tcbot.engine.chain.SingleBuildRunCtx; +import org.apache.ignite.tcbot.engine.conf.ITcBotConfig; import org.apache.ignite.tcbot.engine.defect.BlameCandidate; import org.apache.ignite.tcbot.engine.defect.DefectCompacted; import org.apache.ignite.tcbot.engine.defect.DefectFirstBuild; @@ -63,6 +65,10 @@ import org.apache.ignite.tcignited.build.FatBuildDao; import org.apache.ignite.tcignited.build.ITest; import org.apache.ignite.tcignited.creds.ICredentialsProv; +import org.apache.ignite.tcignited.history.IRunHistory; + +import static org.apache.ignite.tcignited.history.RunStatus.RES_MISSING; +import static org.apache.ignite.tcignited.history.RunStatus.RES_OK; public class BoardService { @Inject IIssuesStorage issuesStorage; @@ -74,6 +80,7 @@ public class BoardService { @Inject IStringCompactor compactor; @Inject BuildChainProcessor buildChainProcessor; @Inject IUserStorage userStorage; + @Inject ITcBotConfig cfg; /** * @param creds Credentials. @@ -118,7 +125,7 @@ public BoardSummaryUi summary(ICredentialsProv creds) { rebuild = !freshRebuild.isEmpty() ? freshRebuild.stream().findFirst() : Optional.empty(); for (DefectIssue issue : cause.issues()) { - BoardDefectIssueUi issueUi = processIssue(tcIgn, rebuild, issue); + BoardDefectIssueUi issueUi = processIssue(tcIgn, rebuild, issue, firstBuild.buildTypeId()); defectUi.addIssue(issueUi); } @@ -134,7 +141,7 @@ public BoardSummaryUi summary(ICredentialsProv creds) { public BoardDefectIssueUi processIssue(ITeamcityIgnited tcIgn, Optional rebuild, - DefectIssue issue) { + DefectIssue issue, int projectId) { Optional testResult; String issueType = compactor.getStringFromId(issue.issueTypeCode()); @@ -176,15 +183,47 @@ public BoardDefectIssueUi processIssue(ITeamcityIgnited tcIgn, if (test.isIgnoredTest() || test.isMutedTest()) status = IssueResolveStatus.IGNORED; + else if (IssueType.newTestWithHighFlakyRate.code().equals(issueType)) { + int fullSuiteNameAndFullTestName = issue.testNameCid(); + + int branchName = rebuild.get().branchName(); + + IRunHistory runStat = tcIgn.getTestRunHist(fullSuiteNameAndFullTestName, projectId, branchName); + + if (runStat == null) + status = IssueResolveStatus.UNKNOWN; + else { + List runResults = runStat.getLatestRunResults(); + if (runResults == null) + status = IssueResolveStatus.UNKNOWN; + else { + int confidenceOkTestsRow = Math.max(1, + (int) Math.ceil(Math.log(1 - cfg.confidence()) / Math.log(1 - issue.getFlakyRate() / 100.0))); + Collections.reverse(runResults); + int okTestRow = 0; + + for (Integer run : runResults) { + if (run == null || run == RES_MISSING.getCode()) + continue; + if (run == RES_OK.getCode() && (okTestRow < confidenceOkTestsRow)) + okTestRow++; + else + break; + } + + status = okTestRow >= confidenceOkTestsRow ? IssueResolveStatus.FIXED : IssueResolveStatus.FAILING; + } + } + } else status = test.isFailedTest(compactor) ? IssueResolveStatus.FAILING : IssueResolveStatus.FIXED; FatBuildCompacted fatBuildCompacted = rebuild.get(); Long testNameId = test.getTestId(); - String projectId = fatBuildCompacted.projectId(compactor); + String RebuildProjectId = fatBuildCompacted.projectId(compactor); String branchName = fatBuildCompacted.branchName(compactor); - webUrl = DsTestFailureUi.buildWebLink(tcIgn, testNameId, projectId, branchName); + webUrl = DsTestFailureUi.buildWebLink(tcIgn, testNameId, RebuildProjectId, branchName); } else { //exception for new test. removal of test means test is fixed @@ -244,6 +283,7 @@ protected String issuesToDefects() { int issueTypeCid = compactor.getStringId(issue.type); Integer testNameCid = compactor.getStringIdIfPresent(testName); int trackedBranchCid = compactor.getStringId(issue.trackedBranchName); + double flakyRate = issue.flakyRate; int tcSrvCodeCid = compactor.getStringId(srvCode); defectStorage.merge(tcSrvCodeCid, srvId, fatBuild, @@ -252,7 +292,7 @@ protected String issuesToDefects() { defect.trackedBranchCidSetIfEmpty(trackedBranchCid); - defect.computeIfAbsent(fatBuild).addIssue(issueTypeCid, testNameCid); + defect.computeIfAbsent(fatBuild).addIssue(issueTypeCid, testNameCid, flakyRate); defect.removeOldVerBlameCandidates(); diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java index 3c9af9a8d..1d38a21f0 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java @@ -30,9 +30,21 @@ public interface ITcBotConfig extends IDataSourcesConfigSupplier { /** Default server code. */ public String DEFAULT_SERVER_CODE = "apache"; + /** Default flaky rate. */ + public Integer DEFAULT_FLAKY_RATE = 20; + + /** Default confidence. */ + public Double DEFAULT_CONFIDENCE = 0.95; + /** */ public String primaryServerCode(); + /** */ + public Integer flakyRate(); + + /** */ + public Double confidence(); + /** * @return Tracked branches configuration for TC Bot. */ diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/TcBotJsonConfig.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/TcBotJsonConfig.java index 818ba5aa2..0fd0f2420 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/TcBotJsonConfig.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/TcBotJsonConfig.java @@ -38,6 +38,12 @@ public class TcBotJsonConfig implements ITrackedBranchesConfig { /** Primary server ID. */ @Nullable private String primaryServerCode; + /** Flaky rate to consider test as a flaky test. */ + @Nullable private Integer flakyRate; + + /** Сonfidence (used with flaky tests). */ + @Nullable private Double confidence; + /** Additional list Servers to be used for validation of PRs, but not for tracking any branches. */ private List tcServers = new ArrayList<>(); @@ -83,6 +89,20 @@ public void addBranch(BranchTracked branch) { return primaryServerCode; } + /** + * @return Flaky rate to consider test as a flaky test. + */ + @Nullable public Integer flakyRate() { + return flakyRate; + } + + /** + * @return Сonfidence. + */ + @Nullable public Double confidence() { + return confidence; + } + public Optional getTcConfig(String code) { return tcServers.stream().filter(s -> Objects.equals(code, s.getCode())).findAny(); } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectFirstBuild.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectFirstBuild.java index fd271ce34..568296f8d 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectFirstBuild.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectFirstBuild.java @@ -33,8 +33,8 @@ public DefectFirstBuild(FatBuildCompacted build) { this.build = build; } - public DefectFirstBuild addIssue(int typeCid, Integer testNameCid) { - issues.add(new DefectIssue(typeCid, testNameCid)); + public DefectFirstBuild addIssue(int typeCid, Integer testNameCid, double flakyRate) { + issues.add(new DefectIssue(typeCid, testNameCid, flakyRate)); return this; } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectIssue.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectIssue.java index 5ee60c7f8..726332fb8 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectIssue.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectIssue.java @@ -24,9 +24,12 @@ public class DefectIssue { private int issueTypeCode; private int testOrSuiteName; - public DefectIssue(int issueTypeCode, Integer testNameCid) { + private double flakyRate; + + public DefectIssue(int issueTypeCode, Integer testNameCid, double flakyRate) { this.issueTypeCode = issueTypeCode; testOrSuiteName = testNameCid; + this.flakyRate = flakyRate; } /** {@inheritDoc} */ @@ -52,4 +55,8 @@ public int testNameCid() { public int issueTypeCode() { return issueTypeCode; } + + public double getFlakyRate() { + return flakyRate; + } } diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/EventTemplates.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/EventTemplates.java index b97947510..415f4815f 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/EventTemplates.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/EventTemplates.java @@ -27,9 +27,11 @@ import static org.apache.ignite.tcignited.history.RunStatus.RES_MISSING; public class EventTemplates { - private static final int OK = RES_OK.getCode(); - private static final int FAIL = RES_FAILURE.getCode(); - private static final int MISSING = RES_MISSING.getCode(); + public static final int OK = RES_OK.getCode(); + public static final int FAIL = RES_FAILURE.getCode(); + public static final int MISSING = RES_MISSING.getCode(); + public static final int OK_OR_FAILURE = RES_OK_OR_FAILURE.getCode(); + public static final int CRITICAL_FAILURE = RES_CRITICAL_FAILURE.getCode(); public static final EventTemplate newFailure = new EventTemplate( new int[]{OK, OK, OK, OK, OK}, @@ -37,8 +39,8 @@ public class EventTemplates { ); public static final EventTemplate newCriticalFailure = new EventTemplate( - new int[]{RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode()}, - new int[]{RES_CRITICAL_FAILURE.getCode(), RES_CRITICAL_FAILURE.getCode(), RES_CRITICAL_FAILURE.getCode(), RES_CRITICAL_FAILURE.getCode()} + new int[]{OK_OR_FAILURE, OK_OR_FAILURE, OK_OR_FAILURE, OK_OR_FAILURE, OK_OR_FAILURE}, + new int[]{CRITICAL_FAILURE, CRITICAL_FAILURE, CRITICAL_FAILURE, CRITICAL_FAILURE} ); public static final EventTemplate newContributedTestFailure = new EventTemplate( diff --git a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssueType.java b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssueType.java index b7b6cce6f..25171c40f 100644 --- a/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssueType.java +++ b/tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssueType.java @@ -34,7 +34,10 @@ public enum IssueType { newCriticalFailure("newCriticalFailure", "New Critical Failure"), /** New trusted suite failure. */ - newTrustedSuiteFailure("newTrustedSuiteFailure", "New Trusted Suite failure"); + newTrustedSuiteFailure("newTrustedSuiteFailure", "New Trusted Suite failure"), + + /** New failure for flaky test. */ + newTestWithHighFlakyRate("newTestWithHighFlakyRate", "Test with high flaky rate"); /** Code. */ private final String code; diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java index ac11fb0f5..92088e60a 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/ci/teamcity/ignited/fatbuild/FatBuildCompacted.java @@ -544,6 +544,10 @@ public String projectId(IStringCompactor compactor) { return compactor.getStringFromId(projectId); } + public int projectId() { + return projectId; + } + public List problems(IStringCompactor compactor) { if (this.problems == null) return Collections.emptyList(); diff --git a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistory.java b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistory.java index b54bc53d4..ff60b0ab3 100644 --- a/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistory.java +++ b/tcbot-teamcity-ignited/src/main/java/org/apache/ignite/tcignited/history/IRunHistory.java @@ -16,18 +16,22 @@ */ package org.apache.ignite.tcignited.history; +import java.util.stream.Stream; import javax.annotation.Nullable; import java.util.List; +import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation; /** * Test or Build run statistics. */ public interface IRunHistory { - /** - * - */ + public boolean isFlaky(); + public Iterable invocations(); + + public Stream getInvocations(); + @Nullable List getLatestRunResults();