Skip to content
Permalink
Browse files
IGNITE-12010: Consider newly contributed test as a blocker if it runs…
… more than 1 minute
  • Loading branch information
dspavlov committed Jul 24, 2019
1 parent ce7f2d3 commit 99933ded617c523c37adeeede5009f4cd2a6f922
Show file tree
Hide file tree
Showing 15 changed files with 194 additions and 131 deletions.
@@ -810,14 +810,7 @@ public Visa notifyJira(
return new Visa("JIRA wasn't commented - no finished builds to analyze." +
" Check builds availabiliy for branch: " + build.branchName + "/" + baseBranch);

blockers = suitesStatuses.stream()
.mapToInt(suite -> {
if (suite.testFailures.isEmpty())
return 1;

return suite.testFailures.size();
})
.sum();
blockers = suitesStatuses.stream().mapToInt(DsSuiteUi::totalBlockers).sum();

String comment = generateJiraComment(suitesStatuses, build.webUrl, buildTypeId, tcIgnited, blockers, build.branchName, baseBranch);

@@ -879,18 +872,7 @@ private String generateJiraComment(List<DsSuiteUi> suites, String webUrl, String
else
res.append(jiraEscText(failure.name));

DsHistoryStatUi recent = failure.histBaseBranch.recent;

if (recent != null) {
if (recent.failureRate != null) {
res.append(" - ").append(recent.failureRate).append("% fails in last ")
.append(recent.runs).append(" ").append(jiraEscText(baseBranchDisp)).append(" runs.");
}
else if (recent.failures != null && recent.runs != null) {
res.append(" - ").append(recent.failures).append(" fails / ")
.append(recent.runs).append(" ").append(jiraEscText(baseBranchDisp)).append(" runs.");
}
}
res.append(" - ").append(jiraEscText(failure.blockerComment));

res.append("\\n");

@@ -28,7 +28,7 @@
public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot";

/** TC Bot Version. */
public static final String VERSION = "20190723";
public static final String VERSION = "20190724";

/** Java version, where Web App is running. */
public String javaVer;
@@ -819,9 +819,14 @@ function showTestFailData(testFail, isFailureShown, settings) {

var haveIssue = isDefinedAndFilled(testFail.webIssueUrl) && isDefinedAndFilled(testFail.webIssueText);

var color = (isFailureShown && failRateDefined)
? failureRateToColor(failRate)
: "white";
var color;
if (testFail.success === true) {
color = "green";
} else {
color = (isFailureShown && failRateDefined)
? failureRateToColor(failRate)
: "white";
}

var investigated = isDefinedAndFilled(testFail.investigated) && testFail.investigated;
if (investigated) {
@@ -29,7 +29,7 @@
},
methods: {
formChanged: function () {
checkForUpdate();
loadData();
}
}
});
@@ -191,7 +191,7 @@

<div id="vueQueryForm">
<v-app id="prQueryForm" name="prQueryForm">
<span class="formgroup" style="margin: 14px">
<div class="formgroup">
<span>Base branch: </span>
<select v-model="baseBranchSelected" @change="formChanged">
<option disabled value="">Please select one</option>
@@ -200,9 +200,9 @@
{{ option }}
</option>
</select>
</span>
&nbsp;&nbsp;
<span id="loadStatus"></span>
&nbsp;&nbsp;
<span id="loadStatus"></span>
</div>
<div id="divFailures"></div>
</v-app>
</div>
@@ -100,7 +100,7 @@ public void testAllBuildsArePresentInMergedBuilds() {
else
assertTrue(suite.failedTests() >= 1);

for (IMultTestOccurrence test : suite.getFailedTests()) {
for (TestCompactedMult test : suite.getFailedTests()) {
if (test.getName().startsWith(UNIQUE_FAILED_TEST))
assertEquals(1, test.failuresCount());
else if (test.getName().equals(TEST_FAILING_EVERY_TIME))
@@ -132,7 +132,7 @@ else if (test.getName().equals(TEST_FAILING_EVERY_TIME))
System.out.println(suite.getFailedTestsNames().collect(Collectors.toList()));

if (suite.suiteName() != null && suite.suiteName().startsWith(UNIQUE_FAILED_TEST)) {
for (IMultTestOccurrence test : suite.getFailedTests())
for (TestCompactedMult test : suite.getFailedTests())
fail("Failure found but should be hidden by re-run " + test.getName());
}
}
@@ -17,6 +17,8 @@

package org.apache.ignite.tcbot.common;

import java.util.concurrent.TimeUnit;

/**
* TC Bot constants: contains magic numbers for project.
*
@@ -46,4 +48,7 @@ public class TcBotConst {

/** Non flaky test failure rate: less that this failure rate in base branch is still blocker border, percents. */
public static final double NON_FLAKY_TEST_FAIL_RATE_BLOCKER_BORDER_PERCENTS = 4.;

/** Max test duration for runall. If duration is greater, (new) test considered as blocker. */
public static final long MAX_NEW_TEST_DURATION_FOR_RUNALL_MS = TimeUnit.MINUTES.toMillis(1);
}
@@ -63,7 +63,13 @@ public String branchName() {
}

public Stream<MultBuildRunCtx> failedChildSuites() {
return suites().filter(MultBuildRunCtx::isFailed);
Predicate<MultBuildRunCtx> filter = MultBuildRunCtx::isFailed;

return filteredChildSuites(filter);
}

public Stream<MultBuildRunCtx> filteredChildSuites(Predicate<MultBuildRunCtx> filter) {
return suites().filter(filter);
}

/**

This file was deleted.

@@ -20,7 +20,6 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.cache.CacheBuilder;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
@@ -33,13 +32,15 @@
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.tcbot.common.util.CollectionUtil;
@@ -71,9 +72,13 @@ public class MultBuildRunCtx implements ISuiteResults {
/** Builds: Single execution. */
private List<SingleBuildRunCtx> builds = new CopyOnWriteArrayList<>();

private final com.google.common.cache.Cache<Integer, Optional<ISuiteRunHistory>> historyCacheMap
/** History cache map. */
private final com.google.common.cache.Cache<Integer, Optional<ISuiteRunHistory>> histCacheMap
= CacheBuilder.newBuilder().build();

/** Tests merged: test name ID -> test compacted */
private volatile Map<Integer, TestCompactedMult> testsMerged = null;

public void addBuild(SingleBuildRunCtx ctx) {
builds.add(ctx);
}
@@ -315,23 +320,18 @@ public Stream<Map.Entry<String, Long>> getTopLogConsumers() {
public Stream<TestCompactedMult> getTopLongRunning() {
Comparator<TestCompactedMult> comparing = Comparator.comparing(TestCompactedMult::getAvgDurationMs);

Map<Integer, TestCompactedMult> res = new HashMap<>();

builds.forEach(singleBuildRunCtx -> {
saveToMap(res, singleBuildRunCtx.getAllTests());
});

return CollectionUtil.top(res.values().stream(), 3, comparing).stream();
return CollectionUtil.top(getTestsMerged().values().stream(), 3, comparing).stream();
}

public List<TestCompactedMult> getFailedTests() {
Map<Integer, TestCompactedMult> res = new HashMap<>();

builds.forEach(singleBuildRunCtx -> {
saveToMap(res, singleBuildRunCtx.getFailedNotMutedTests());
});
return getFilteredTests(TestCompactedMult::isFailedButNotMuted);
}

return new ArrayList<>(res.values());
public List<TestCompactedMult> getFilteredTests(Predicate<TestCompactedMult> filter) {
return getTestsMerged().values()
.stream()
.filter(filter)
.collect(Collectors.toList());
}

public void saveToMap(Map<Integer, TestCompactedMult> res, Stream<TestCompacted> tests) {
@@ -608,23 +608,48 @@ public int totalTests() {
return (int)buildsStream().mapToInt(SingleBuildRunCtx::totalNotMutedTests).average().orElse(0.0);
}

public int trustedTests(ITeamcityIgnited tcIgnited,
@Nullable Integer branchName) {
/**
* @return all tests grouped by its name
*/
private Map<Integer, TestCompactedMult> getTestsMerged() {
if (testsMerged == null) {
synchronized (this) {
if (testsMerged == null) {
HashMap<Integer, TestCompactedMult> res = new HashMap<>();

builds.forEach(singleBuildRunCtx -> {
saveToMap(res, singleBuildRunCtx.getAllTests());
});

testsMerged = res;
}
}
}
return testsMerged;
}

/**
* @param tcIgnited Tc ignited.
* @param branchName Branch name.
*/
public int trustedTests(ITeamcityIgnited tcIgnited, @Nullable Integer branchName) {
AtomicInteger trustedCnt = new AtomicInteger();
Map<Integer, TestCompactedMult> res = new HashMap<>();

//todo can cache mult occurrences in ctx
builds.forEach(singleBuildRunCtx -> {
saveToMap(res,
singleBuildRunCtx.getAllTests().filter(t -> !t.isIgnoredTest() && !t.isMutedTest()));
});
getFilteredTests(t -> !t.isMutedOrIgored()).forEach((testMult) -> {
IRunHistory baseBranchStat = testMult.history(tcIgnited, branchName);

boolean testWillBeBlockerIfFailed = false;

res.forEach((testNameId, compactedMult) -> {
IRunHistory stat = compactedMult.history(tcIgnited, branchName);
String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
boolean b = testBlockerComment != null;
if (b) // this test will be considered as blocker if will fail
if (baseBranchStat == null)
testWillBeBlockerIfFailed = true;

float failRate = baseBranchStat.getFailRate();
boolean lowFailureRate = failRate * 100.0f < TcBotConst.NON_FLAKY_TEST_FAIL_RATE_BLOCKER_BORDER_PERCENTS;

if (lowFailureRate && !baseBranchStat.isFlaky())
testWillBeBlockerIfFailed = true;

if (testWillBeBlockerIfFailed) // this test will be considered as blocker if will fail
trustedCnt.addAndGet(1);
});

@@ -667,7 +692,7 @@ ISuiteRunHistory suiteHist(ITeamcityIgnited tcIgn, @Nullable Integer baseBranchI
return null;

try {
return historyCacheMap.get(baseBranchId,
return histCacheMap.get(baseBranchId,
() -> {
return Optional.ofNullable(tcIgn.getSuiteRunHist(buildTypeIdId, baseBranchId));
})
@@ -677,4 +702,8 @@ ISuiteRunHistory suiteHist(ITeamcityIgnited tcIgn, @Nullable Integer baseBranchI
throw ExceptionUtil.propagateException(e);
}
}

public boolean hasTestToReport(ITeamcityIgnited tcIgnited, Integer baseBranchId) {
return !getFilteredTests(test -> test.includeIntoReport(tcIgnited, baseBranchId)).isEmpty();
}
}

0 comments on commit 99933de

Please sign in to comment.