Skip to content
Permalink
Browse files
Show tests on TC board that were stable and have become flaky (#159)
Signed-off-by: Ivan Rakov <ivan.glukos@gmail.com>
  • Loading branch information
sergeyuttsel authored and glukos committed Apr 27, 2020
1 parent 81c9280 commit 12a535e1477c1cbeae8b8153e98b5227a9043eb8
Showing 14 changed files with 182 additions and 26 deletions.
@@ -114,6 +114,20 @@ protected TcBotJsonConfig getConfig() {
return Strings.isNullOrEmpty(srvCode) ? ITcBotConfig.DEFAULT_SERVER_CODE : srvCode;
}

/** {@inheritDoc} */
@Override public Integer flakyRate() {
Integer flakyRate = getConfig().flakyRate();

return flakyRate == null || flakyRate < 0 || flakyRate > 100 ? ITcBotConfig.DEFAULT_FLAKY_RATE : flakyRate;
}

/** {@inheritDoc} */
@Override public Double confidence() {
Double confidence = getConfig().confidence();

return confidence == null || confidence < 0 || confidence > 1 ? ITcBotConfig.DEFAULT_CONFIDENCE : confidence;
}

@Override
public ITrackedBranchesConfig getTrackedBranches() {
return getConfig();
@@ -30,11 +30,13 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.ci.issue.Issue;
import org.apache.ignite.ci.issue.IssueKey;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
import org.apache.ignite.tcbot.engine.issue.IssueType;
import org.apache.ignite.ci.jobs.CheckQueueJob;
@@ -64,11 +66,14 @@
import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
import org.apache.ignite.tcignited.SyncMode;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.InvocationData;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.ignite.tcignited.buildref.BranchEquivalence.normalizeBranch;
import static org.apache.ignite.tcignited.history.RunStatus.RES_FAILURE;
import static org.apache.ignite.tcignited.history.RunStatus.RES_OK;

/**
*
@@ -471,15 +476,49 @@ private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited,
type = IssueType.newFailure;
final String flakyComments = runStat.getFlakyComments();

if (!Strings.isNullOrEmpty(flakyComments)) {
if (runStat.detectTemplate(EventTemplates.newFailureForFlakyTest) == null) {
logger.info("Skipping registering new issue for test fail:" +
" Test seems to be flaky " + name + ": " + flakyComments);
if (!Strings.isNullOrEmpty(flakyComments) &&
runStat.detectTemplate(EventTemplates.newFailureForFlakyTest) != null)
type = IssueType.newFailureForFlakyTest;
}
}

firstFailedBuildId = null;
}
double flakyRate = 0;

if (firstFailedBuildId == null || type == null) {
List<Invocation> invocations = runStat.getInvocations().
filter(invocation -> invocation != null && invocation.status() != InvocationData.MISSING)
.collect(Collectors.toList());

int confidenceOkTestsRow = Math.max(1,
(int) Math.ceil(Math.log(1 - cfg.confidence()) / Math.log(1 - cfg.flakyRate() / 100.0)));

if (invocations.size() >= confidenceOkTestsRow * 2) {
List<Invocation> lastInvocations =
invocations.subList(invocations.size() - confidenceOkTestsRow * 2, invocations.size());

int stableTestRuns = 0;

for (int i = 0; i < confidenceOkTestsRow; i++) {
if (lastInvocations.get(i).status() == RES_OK.getCode())
stableTestRuns++;
else
type = IssueType.newFailureForFlakyTest;
break;
}

if (stableTestRuns == confidenceOkTestsRow) {
long failedTestRuns = 0;

for (int i = confidenceOkTestsRow; i < confidenceOkTestsRow * 2; i++) {
if (lastInvocations.get(i).status() == RES_FAILURE.getCode())
failedTestRuns++;
}

flakyRate = (double) failedTestRuns / confidenceOkTestsRow * 100;

if (flakyRate > cfg.flakyRate()) {
type = IssueType.newTestWithHighFlakyRate;
firstFailedBuildId = lastInvocations.get(confidenceOkTestsRow).buildId();
}
}
}
}
@@ -501,6 +540,7 @@ private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited,
issue.trackedBranchName = trackedBranch;
issue.displayName = testFailure.testName;
issue.webUrl = testFailure.webUrl;
issue.flakyRate = flakyRate;

issue.buildTags.addAll(suiteTags);

@@ -98,6 +98,14 @@ public MockBasedTcBotModule() {
return ITcBotConfig.DEFAULT_SERVER_CODE;
}

@Override public Integer flakyRate() {
return DEFAULT_FLAKY_RATE;
}

@Override public Double confidence() {
return DEFAULT_CONFIDENCE;
}

@Override public ITrackedBranchesConfig getTrackedBranches() {
return tracked;
}
@@ -106,8 +106,8 @@ public void testDetector() {

Map<String, String> buildWoChanges = new TreeMap<String, String>() {
{
put("testFailedShoudlBeConsideredAsFlaky", "0000011111");
put("testFlakyStableFailure", "0000011111111111");
put("testFailedShouldBeConsideredAsFlaky", "0000011111");
put("testFlakyStableFailure", "0000010101100101");
}
};

@@ -47,6 +47,8 @@ public class Issue {
/** Display type. for issue. Kept for backward compatibilty with older records without type code. */
private String displayType;

public double flakyRate;

@Nullable
public String trackedBranchName;

@@ -19,6 +19,7 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -43,6 +44,7 @@
import org.apache.ignite.tcbot.common.util.FutureUtil;
import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.chain.SingleBuildRunCtx;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.defect.BlameCandidate;
import org.apache.ignite.tcbot.engine.defect.DefectCompacted;
import org.apache.ignite.tcbot.engine.defect.DefectFirstBuild;
@@ -63,6 +65,10 @@
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.build.ITest;
import org.apache.ignite.tcignited.creds.ICredentialsProv;
import org.apache.ignite.tcignited.history.IRunHistory;

import static org.apache.ignite.tcignited.history.RunStatus.RES_MISSING;
import static org.apache.ignite.tcignited.history.RunStatus.RES_OK;

public class BoardService {
@Inject IIssuesStorage issuesStorage;
@@ -74,6 +80,7 @@ public class BoardService {
@Inject IStringCompactor compactor;
@Inject BuildChainProcessor buildChainProcessor;
@Inject IUserStorage userStorage;
@Inject ITcBotConfig cfg;

/**
* @param creds Credentials.
@@ -118,7 +125,7 @@ public BoardSummaryUi summary(ICredentialsProv creds) {
rebuild = !freshRebuild.isEmpty() ? freshRebuild.stream().findFirst() : Optional.empty();

for (DefectIssue issue : cause.issues()) {
BoardDefectIssueUi issueUi = processIssue(tcIgn, rebuild, issue);
BoardDefectIssueUi issueUi = processIssue(tcIgn, rebuild, issue, firstBuild.buildTypeId());

defectUi.addIssue(issueUi);
}
@@ -134,7 +141,7 @@ public BoardSummaryUi summary(ICredentialsProv creds) {

public BoardDefectIssueUi processIssue(ITeamcityIgnited tcIgn,
Optional<FatBuildCompacted> rebuild,
DefectIssue issue) {
DefectIssue issue, int projectId) {
Optional<ITest> testResult;

String issueType = compactor.getStringFromId(issue.issueTypeCode());
@@ -176,15 +183,47 @@ public BoardDefectIssueUi processIssue(ITeamcityIgnited tcIgn,

if (test.isIgnoredTest() || test.isMutedTest())
status = IssueResolveStatus.IGNORED;
else if (IssueType.newTestWithHighFlakyRate.code().equals(issueType)) {
int fullSuiteNameAndFullTestName = issue.testNameCid();

int branchName = rebuild.get().branchName();

IRunHistory runStat = tcIgn.getTestRunHist(fullSuiteNameAndFullTestName, projectId, branchName);

if (runStat == null)
status = IssueResolveStatus.UNKNOWN;
else {
List<Integer> runResults = runStat.getLatestRunResults();
if (runResults == null)
status = IssueResolveStatus.UNKNOWN;
else {
int confidenceOkTestsRow = Math.max(1,
(int) Math.ceil(Math.log(1 - cfg.confidence()) / Math.log(1 - issue.getFlakyRate() / 100.0)));
Collections.reverse(runResults);
int okTestRow = 0;

for (Integer run : runResults) {
if (run == null || run == RES_MISSING.getCode())
continue;
if (run == RES_OK.getCode() && (okTestRow < confidenceOkTestsRow))
okTestRow++;
else
break;
}

status = okTestRow >= confidenceOkTestsRow ? IssueResolveStatus.FIXED : IssueResolveStatus.FAILING;
}
}
}
else
status = test.isFailedTest(compactor) ? IssueResolveStatus.FAILING : IssueResolveStatus.FIXED;

FatBuildCompacted fatBuildCompacted = rebuild.get();
Long testNameId = test.getTestId();
String projectId = fatBuildCompacted.projectId(compactor);
String RebuildProjectId = fatBuildCompacted.projectId(compactor);
String branchName = fatBuildCompacted.branchName(compactor);

webUrl = DsTestFailureUi.buildWebLink(tcIgn, testNameId, projectId, branchName);
webUrl = DsTestFailureUi.buildWebLink(tcIgn, testNameId, RebuildProjectId, branchName);
}
else {
//exception for new test. removal of test means test is fixed
@@ -244,6 +283,7 @@ protected String issuesToDefects() {
int issueTypeCid = compactor.getStringId(issue.type);
Integer testNameCid = compactor.getStringIdIfPresent(testName);
int trackedBranchCid = compactor.getStringId(issue.trackedBranchName);
double flakyRate = issue.flakyRate;

int tcSrvCodeCid = compactor.getStringId(srvCode);
defectStorage.merge(tcSrvCodeCid, srvId, fatBuild,
@@ -252,7 +292,7 @@ protected String issuesToDefects() {

defect.trackedBranchCidSetIfEmpty(trackedBranchCid);

defect.computeIfAbsent(fatBuild).addIssue(issueTypeCid, testNameCid);
defect.computeIfAbsent(fatBuild).addIssue(issueTypeCid, testNameCid, flakyRate);

defect.removeOldVerBlameCandidates();

@@ -30,9 +30,21 @@ public interface ITcBotConfig extends IDataSourcesConfigSupplier {
/** Default server code. */
public String DEFAULT_SERVER_CODE = "apache";

/** Default flaky rate. */
public Integer DEFAULT_FLAKY_RATE = 20;

/** Default confidence. */
public Double DEFAULT_CONFIDENCE = 0.95;

/** */
public String primaryServerCode();

/** */
public Integer flakyRate();

/** */
public Double confidence();

/**
* @return Tracked branches configuration for TC Bot.
*/
@@ -38,6 +38,12 @@ public class TcBotJsonConfig implements ITrackedBranchesConfig {
/** Primary server ID. */
@Nullable private String primaryServerCode;

/** Flaky rate to consider test as a flaky test. */
@Nullable private Integer flakyRate;

/** Сonfidence (used with flaky tests). */
@Nullable private Double confidence;

/** Additional list Servers to be used for validation of PRs, but not for tracking any branches. */
private List<TcServerConfig> tcServers = new ArrayList<>();

@@ -83,6 +89,20 @@ public void addBranch(BranchTracked branch) {
return primaryServerCode;
}

/**
* @return Flaky rate to consider test as a flaky test.
*/
@Nullable public Integer flakyRate() {
return flakyRate;
}

/**
* @return Сonfidence.
*/
@Nullable public Double confidence() {
return confidence;
}

public Optional<TcServerConfig> getTcConfig(String code) {
return tcServers.stream().filter(s -> Objects.equals(code, s.getCode())).findAny();
}
@@ -33,8 +33,8 @@ public DefectFirstBuild(FatBuildCompacted build) {
this.build = build;
}

public DefectFirstBuild addIssue(int typeCid, Integer testNameCid) {
issues.add(new DefectIssue(typeCid, testNameCid));
public DefectFirstBuild addIssue(int typeCid, Integer testNameCid, double flakyRate) {
issues.add(new DefectIssue(typeCid, testNameCid, flakyRate));

return this;
}
@@ -24,9 +24,12 @@ public class DefectIssue {
private int issueTypeCode;
private int testOrSuiteName;

public DefectIssue(int issueTypeCode, Integer testNameCid) {
private double flakyRate;

public DefectIssue(int issueTypeCode, Integer testNameCid, double flakyRate) {
this.issueTypeCode = issueTypeCode;
testOrSuiteName = testNameCid;
this.flakyRate = flakyRate;
}

/** {@inheritDoc} */
@@ -52,4 +55,8 @@ public int testNameCid() {
public int issueTypeCode() {
return issueTypeCode;
}

public double getFlakyRate() {
return flakyRate;
}
}
@@ -27,18 +27,20 @@
import static org.apache.ignite.tcignited.history.RunStatus.RES_MISSING;

public class EventTemplates {
private static final int OK = RES_OK.getCode();
private static final int FAIL = RES_FAILURE.getCode();
private static final int MISSING = RES_MISSING.getCode();
public static final int OK = RES_OK.getCode();
public static final int FAIL = RES_FAILURE.getCode();
public static final int MISSING = RES_MISSING.getCode();
public static final int OK_OR_FAILURE = RES_OK_OR_FAILURE.getCode();
public static final int CRITICAL_FAILURE = RES_CRITICAL_FAILURE.getCode();

public static final EventTemplate newFailure = new EventTemplate(
new int[]{OK, OK, OK, OK, OK},
new int[]{FAIL, FAIL, FAIL, FAIL}
);

public static final EventTemplate newCriticalFailure = new EventTemplate(
new int[]{RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode(), RES_OK_OR_FAILURE.getCode()},
new int[]{RES_CRITICAL_FAILURE.getCode(), RES_CRITICAL_FAILURE.getCode(), RES_CRITICAL_FAILURE.getCode(), RES_CRITICAL_FAILURE.getCode()}
new int[]{OK_OR_FAILURE, OK_OR_FAILURE, OK_OR_FAILURE, OK_OR_FAILURE, OK_OR_FAILURE},
new int[]{CRITICAL_FAILURE, CRITICAL_FAILURE, CRITICAL_FAILURE, CRITICAL_FAILURE}
);

public static final EventTemplate newContributedTestFailure = new EventTemplate(
@@ -34,7 +34,10 @@ public enum IssueType {
newCriticalFailure("newCriticalFailure", "New Critical Failure"),

/** New trusted suite failure. */
newTrustedSuiteFailure("newTrustedSuiteFailure", "New Trusted Suite failure");
newTrustedSuiteFailure("newTrustedSuiteFailure", "New Trusted Suite failure"),

/** New failure for flaky test. */
newTestWithHighFlakyRate("newTestWithHighFlakyRate", "Test with high flaky rate");

/** Code. */
private final String code;

0 comments on commit 12a535e

Please sign in to comment.