Skip to content

Commit

Permalink
Show tests on TC board that were stable and have become flaky (#159)
Browse files Browse the repository at this point in the history 
Signed-off-by: Ivan Rakov <ivan.glukos@gmail.com>
  • Loading branch information
@sergeyuttsel @glukos
sergeyuttsel authored and glukos committed Apr 27, 2020
1 parent 81c9280 commit 12a535e
Show file tree
Hide file tree
Showing 14 changed files with 182 additions and 26 deletions.
14 changes: 14 additions & 0 deletions ...te-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/conf/LocalFilesBasedConfig.java
View file
Expand Up @@ -114,6 +114,20 @@ protected TcBotJsonConfig getConfig() {
return Strings.isNullOrEmpty(srvCode) ? ITcBotConfig.DEFAULT_SERVER_CODE : srvCode;
}

/** {@inheritDoc} */
@Override public Integer flakyRate() {
Integer flakyRate = getConfig().flakyRate();

return flakyRate == null || flakyRate < 0 || flakyRate > 100 ? ITcBotConfig.DEFAULT_FLAKY_RATE : flakyRate;
}

/** {@inheritDoc} */
@Override public Double confidence() {
Double confidence = getConfig().confidence();

return confidence == null || confidence < 0 || confidence > 1 ? ITcBotConfig.DEFAULT_CONFIDENCE : confidence;
}

@Override
public ITrackedBranchesConfig getTrackedBranches() {
return getConfig();
Expand Down
54 changes: 47 additions & 7 deletions ignite-tc-helper-web/src/main/java/org/apache/ignite/ci/tcbot/issue/IssueDetector.java
View file
Expand Up @@ -30,11 +30,13 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.ci.issue.Issue;
import org.apache.ignite.ci.issue.IssueKey;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
import org.apache.ignite.tcbot.engine.issue.IssueType;
import org.apache.ignite.ci.jobs.CheckQueueJob;
Expand Down Expand Up @@ -64,11 +66,14 @@
import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
import org.apache.ignite.tcignited.SyncMode;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.InvocationData;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.ignite.tcignited.buildref.BranchEquivalence.normalizeBranch;
import static org.apache.ignite.tcignited.history.RunStatus.RES_FAILURE;
import static org.apache.ignite.tcignited.history.RunStatus.RES_OK;

/**
*
Expand Down Expand Up @@ -471,15 +476,49 @@ private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited,
type = IssueType.newFailure;
final String flakyComments = runStat.getFlakyComments();

if (!Strings.isNullOrEmpty(flakyComments)) {
if (runStat.detectTemplate(EventTemplates.newFailureForFlakyTest) == null) {
logger.info("Skipping registering new issue for test fail:" +
" Test seems to be flaky " + name + ": " + flakyComments);
if (!Strings.isNullOrEmpty(flakyComments) &&
runStat.detectTemplate(EventTemplates.newFailureForFlakyTest) != null)
type = IssueType.newFailureForFlakyTest;
}
}

firstFailedBuildId = null;
}
double flakyRate = 0;

if (firstFailedBuildId == null || type == null) {
List<Invocation> invocations = runStat.getInvocations().
filter(invocation -> invocation != null && invocation.status() != InvocationData.MISSING)
.collect(Collectors.toList());

int confidenceOkTestsRow = Math.max(1,
(int) Math.ceil(Math.log(1 - cfg.confidence()) / Math.log(1 - cfg.flakyRate() / 100.0)));

if (invocations.size() >= confidenceOkTestsRow * 2) {
List<Invocation> lastInvocations =
invocations.subList(invocations.size() - confidenceOkTestsRow * 2, invocations.size());

int stableTestRuns = 0;

for (int i = 0; i < confidenceOkTestsRow; i++) {
if (lastInvocations.get(i).status() == RES_OK.getCode())
stableTestRuns++;
else
type = IssueType.newFailureForFlakyTest;
break;
}

if (stableTestRuns == confidenceOkTestsRow) {
long failedTestRuns = 0;

for (int i = confidenceOkTestsRow; i < confidenceOkTestsRow * 2; i++) {
if (lastInvocations.get(i).status() == RES_FAILURE.getCode())
failedTestRuns++;
}

flakyRate = (double) failedTestRuns / confidenceOkTestsRow * 100;

if (flakyRate > cfg.flakyRate()) {
type = IssueType.newTestWithHighFlakyRate;
firstFailedBuildId = lastInvocations.get(confidenceOkTestsRow).buildId();
}
}
}
}
Expand All @@ -501,6 +540,7 @@ private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited,
issue.trackedBranchName = trackedBranch;
issue.displayName = testFailure.testName;
issue.webUrl = testFailure.webUrl;
issue.flakyRate = flakyRate;

issue.buildTags.addAll(suiteTags);

Expand Down
8 changes: 8 additions & 0 deletions ...te-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/chain/MockBasedTcBotModule.java
View file
Expand Up @@ -98,6 +98,14 @@ public MockBasedTcBotModule() {
return ITcBotConfig.DEFAULT_SERVER_CODE;
}

@Override public Integer flakyRate() {
return DEFAULT_FLAKY_RATE;
}

@Override public Double confidence() {
return DEFAULT_CONFIDENCE;
}

@Override public ITrackedBranchesConfig getTrackedBranches() {
return tracked;
}
Expand Down
4 changes: 2 additions & 2 deletions ignite-tc-helper-web/src/test/java/org/apache/ignite/ci/tcbot/issue/IssueDetectorTest.java
View file
Expand Up @@ -106,8 +106,8 @@ public void testDetector() {

Map<String, String> buildWoChanges = new TreeMap<String, String>() {
{
put("testFailedShoudlBeConsideredAsFlaky", "0000011111");
put("testFlakyStableFailure", "0000011111111111");
put("testFailedShouldBeConsideredAsFlaky", "0000011111");
put("testFlakyStableFailure", "0000010101100101");
}
};

Expand Down
2 changes: 2 additions & 0 deletions tcbot-engine/src/main/java/org/apache/ignite/ci/issue/Issue.java
View file
Expand Up @@ -47,6 +47,8 @@ public class Issue {
/** Display type. for issue. Kept for backward compatibilty with older records without type code. */
private String displayType;

public double flakyRate;

@Nullable
public String trackedBranchName;

Expand Down
50 changes: 45 additions & 5 deletions tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/board/BoardService.java
View file
Expand Up @@ -19,6 +19,7 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
Expand All @@ -43,6 +44,7 @@
import org.apache.ignite.tcbot.common.util.FutureUtil;
import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.chain.SingleBuildRunCtx;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.defect.BlameCandidate;
import org.apache.ignite.tcbot.engine.defect.DefectCompacted;
import org.apache.ignite.tcbot.engine.defect.DefectFirstBuild;
Expand All @@ -63,6 +65,10 @@
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ITest;
import org.apache.ignite.tcignited.creds.ICredentialsProv;
import org.apache.ignite.tcignited.history.IRunHistory;

import static org.apache.ignite.tcignited.history.RunStatus.RES_MISSING;
import static org.apache.ignite.tcignited.history.RunStatus.RES_OK;

public class BoardService {
@Inject IIssuesStorage issuesStorage;
Expand All @@ -74,6 +80,7 @@ public class BoardService {
@Inject IStringCompactor compactor;
@Inject BuildChainProcessor buildChainProcessor;
@Inject IUserStorage userStorage;
@Inject ITcBotConfig cfg;

/**
* @param creds Credentials.
Expand Down Expand Up @@ -118,7 +125,7 @@ public BoardSummaryUi summary(ICredentialsProv creds) {
rebuild = !freshRebuild.isEmpty() ? freshRebuild.stream().findFirst() : Optional.empty();

for (DefectIssue issue : cause.issues()) {
BoardDefectIssueUi issueUi = processIssue(tcIgn, rebuild, issue);
BoardDefectIssueUi issueUi = processIssue(tcIgn, rebuild, issue, firstBuild.buildTypeId());

defectUi.addIssue(issueUi);
}
Expand All @@ -134,7 +141,7 @@ public BoardSummaryUi summary(ICredentialsProv creds) {

public BoardDefectIssueUi processIssue(ITeamcityIgnited tcIgn,
Optional<FatBuildCompacted> rebuild,
DefectIssue issue) {
DefectIssue issue, int projectId) {
Optional<ITest> testResult;

String issueType = compactor.getStringFromId(issue.issueTypeCode());
Expand Down Expand Up @@ -176,15 +183,47 @@ public BoardDefectIssueUi processIssue(ITeamcityIgnited tcIgn,

if (test.isIgnoredTest() || test.isMutedTest())
status = IssueResolveStatus.IGNORED;
else if (IssueType.newTestWithHighFlakyRate.code().equals(issueType)) {
int fullSuiteNameAndFullTestName = issue.testNameCid();

int branchName = rebuild.get().branchName();

IRunHistory runStat = tcIgn.getTestRunHist(fullSuiteNameAndFullTestName, projectId, branchName);

if (runStat == null)
status = IssueResolveStatus.UNKNOWN;
else {
List<Integer> runResults = runStat.getLatestRunResults();
if (runResults == null)
status = IssueResolveStatus.UNKNOWN;
else {
int confidenceOkTestsRow = Math.max(1,
(int) Math.ceil(Math.log(1 - cfg.confidence()) / Math.log(1 - issue.getFlakyRate() / 100.0)));
Collections.reverse(runResults);
int okTestRow = 0;

for (Integer run : runResults) {
if (run == null || run == RES_MISSING.getCode())
continue;
if (run == RES_OK.getCode() && (okTestRow < confidenceOkTestsRow))
okTestRow++;
else
break;
}

status = okTestRow >= confidenceOkTestsRow ? IssueResolveStatus.FIXED : IssueResolveStatus.FAILING;
}
}
}
else
status = test.isFailedTest(compactor) ? IssueResolveStatus.FAILING : IssueResolveStatus.FIXED;

FatBuildCompacted fatBuildCompacted = rebuild.get();
Long testNameId = test.getTestId();
String projectId = fatBuildCompacted.projectId(compactor);
String RebuildProjectId = fatBuildCompacted.projectId(compactor);
String branchName = fatBuildCompacted.branchName(compactor);

webUrl = DsTestFailureUi.buildWebLink(tcIgn, testNameId, projectId, branchName);
webUrl = DsTestFailureUi.buildWebLink(tcIgn, testNameId, RebuildProjectId, branchName);
}
else {
//exception for new test. removal of test means test is fixed
Expand Down Expand Up @@ -244,6 +283,7 @@ protected String issuesToDefects() {
int issueTypeCid = compactor.getStringId(issue.type);
Integer testNameCid = compactor.getStringIdIfPresent(testName);
int trackedBranchCid = compactor.getStringId(issue.trackedBranchName);
double flakyRate = issue.flakyRate;

int tcSrvCodeCid = compactor.getStringId(srvCode);
defectStorage.merge(tcSrvCodeCid, srvId, fatBuild,
Expand All @@ -252,7 +292,7 @@ protected String issuesToDefects() {

defect.trackedBranchCidSetIfEmpty(trackedBranchCid);

defect.computeIfAbsent(fatBuild).addIssue(issueTypeCid, testNameCid);
defect.computeIfAbsent(fatBuild).addIssue(issueTypeCid, testNameCid, flakyRate);

defect.removeOldVerBlameCandidates();

Expand Down
12 changes: 12 additions & 0 deletions tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/ITcBotConfig.java
View file
Expand Up @@ -30,9 +30,21 @@ public interface ITcBotConfig extends IDataSourcesConfigSupplier {
/** Default server code. */
public String DEFAULT_SERVER_CODE = "apache";

/** Default flaky rate. */
public Integer DEFAULT_FLAKY_RATE = 20;

/** Default confidence. */
public Double DEFAULT_CONFIDENCE = 0.95;

/** */
public String primaryServerCode();

/** */
public Integer flakyRate();

/** */
public Double confidence();

/**
* @return Tracked branches configuration for TC Bot.
*/
Expand Down
20 changes: 20 additions & 0 deletions tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/conf/TcBotJsonConfig.java
View file
Expand Up @@ -38,6 +38,12 @@ public class TcBotJsonConfig implements ITrackedBranchesConfig {
/** Primary server ID. */
@Nullable private String primaryServerCode;

/** Flaky rate to consider test as a flaky test. */
@Nullable private Integer flakyRate;

/** Сonfidence (used with flaky tests). */
@Nullable private Double confidence;

/** Additional list Servers to be used for validation of PRs, but not for tracking any branches. */
private List<TcServerConfig> tcServers = new ArrayList<>();

Expand Down Expand Up @@ -83,6 +89,20 @@ public void addBranch(BranchTracked branch) {
return primaryServerCode;
}

/**
* @return Flaky rate to consider test as a flaky test.
*/
@Nullable public Integer flakyRate() {
return flakyRate;
}

/**
* @return Сonfidence.
*/
@Nullable public Double confidence() {
return confidence;
}

public Optional<TcServerConfig> getTcConfig(String code) {
return tcServers.stream().filter(s -> Objects.equals(code, s.getCode())).findAny();
}
Expand Down
4 changes: 2 additions & 2 deletions tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectFirstBuild.java
View file
Expand Up @@ -33,8 +33,8 @@ public DefectFirstBuild(FatBuildCompacted build) {
this.build = build;
}

public DefectFirstBuild addIssue(int typeCid, Integer testNameCid) {
issues.add(new DefectIssue(typeCid, testNameCid));
public DefectFirstBuild addIssue(int typeCid, Integer testNameCid, double flakyRate) {
issues.add(new DefectIssue(typeCid, testNameCid, flakyRate));

return this;
}
Expand Down
9 changes: 8 additions & 1 deletion tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/defect/DefectIssue.java
View file
Expand Up @@ -24,9 +24,12 @@ public class DefectIssue {
private int issueTypeCode;
private int testOrSuiteName;

public DefectIssue(int issueTypeCode, Integer testNameCid) {
private double flakyRate;

public DefectIssue(int issueTypeCode, Integer testNameCid, double flakyRate) {
this.issueTypeCode = issueTypeCode;
testOrSuiteName = testNameCid;
this.flakyRate = flakyRate;
}

/** {@inheritDoc} */
Expand All @@ -52,4 +55,8 @@ public int testNameCid() {
public int issueTypeCode() {
return issueTypeCode;
}

public double getFlakyRate() {
return flakyRate;
}
}
12 changes: 7 additions & 5 deletions tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/EventTemplates.java
View file
Expand Up @@ -27,18 +27,20 @@
import static org.apache.ignite.tcignited.history.RunStatus.RES_MISSING;

public class EventTemplates {
private static final int OK = RES_OK.getCode();
private static final int FAIL = RES_FAILURE.getCode();
private static final int MISSING = RES_MISSING.getCode();
public static final int OK = RES_OK.getCode();
public static final int FAIL = RES_FAILURE.getCode();
public static final int MISSING = RES_MISSING.getCode();
public static final int OK_OR_FAILURE = RES_OK_OR_FAILURE.getCode();
public static final int CRITICAL_FAILURE = RES_CRITICAL_FAILURE.getCode();

public static final EventTemplate newFailure = new EventTemplate(
new int[]{OK, OK, OK, OK, OK},
new int[]{FAIL, FAIL, FAIL, FAIL}
);

public static final EventTemplate newCriticalFailure = new EventTemplate(
new int[]{RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode()},
new int[]{RES_CRITICAL_FAILURE.getCode(), RES_CRITICAL_FAILURE.getCode(), RES_CRITICAL_FAILURE.getCode(), RES_CRITICAL_FAILURE.getCode()}
new int[]{OK_OR_FAILURE, OK_OR_FAILURE, OK_OR_FAILURE, OK_OR_FAILURE, OK_OR_FAILURE},
new int[]{CRITICAL_FAILURE, CRITICAL_FAILURE, CRITICAL_FAILURE, CRITICAL_FAILURE}
);

public static final EventTemplate newContributedTestFailure = new EventTemplate(
Expand Down
5 changes: 4 additions & 1 deletion tcbot-engine/src/main/java/org/apache/ignite/tcbot/engine/issue/IssueType.java
View file
Expand Up @@ -34,7 +34,10 @@ public enum IssueType {
newCriticalFailure("newCriticalFailure", "New Critical Failure"),

/** New trusted suite failure. */
newTrustedSuiteFailure("newTrustedSuiteFailure", "New Trusted Suite failure");
newTrustedSuiteFailure("newTrustedSuiteFailure", "New Trusted Suite failure"),

/** New failure for flaky test. */
newTestWithHighFlakyRate("newTestWithHighFlakyRate", "Test with high flaky rate");

/** Code. */
private final String code;
Expand Down

0 comments on commit 12a535e

Please sign in to comment.