Skip to content
Permalink
Browse files
Trusted and total tests count: initial implementation (#128)
  • Loading branch information
dspavlov committed Jun 7, 2019
1 parent 9b1d064 commit 912f8f50502b5f61604cf5f01b1b76df44a2ac1a
Showing 44 changed files with 793 additions and 223 deletions.
@@ -33,6 +33,4 @@ public interface IMultTestOccurrence {
public long getAvgDurationMs();

Iterable<TestOccurrenceFull> getOccurrences();

String getPossibleBlockerComment(IRunHistory baseBranchStat);
}
@@ -42,21 +42,6 @@ public interface ISuiteResults {
/** */
public boolean hasBuildMessageProblem();

default boolean hasCriticalProblem() {
return hasJvmCrashProblem()
|| hasTimeoutProblem()
|| hasCompilationProblem()
|| hasMetricProblem();
}

default boolean hasSuiteIncompleteFailure() {
return hasJvmCrashProblem()
|| hasTimeoutProblem()
|| hasOomeProblem()
|| hasExitCodeProblem()
|| hasCompilationProblem()
|| hasMetricProblem();
}

public String suiteId();
}
@@ -18,12 +18,28 @@
package org.apache.ignite.ci.analysis;

import com.google.common.base.Strings;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.OptionalDouble;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nonnull;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.util.CollectionUtil;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.buildlog.ILogCheckResult;
import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
import org.apache.ignite.tcignited.history.IRunHistory;
@@ -33,24 +49,19 @@
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import javax.annotation.Nonnull;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import java.util.stream.Stream;

/**
* Run configuration execution results loaded from different API URLs. Includes tests and problem occurrences; if logs
* processing is done also contains last started test
*/
public class MultBuildRunCtx implements ISuiteResults {
/** Cancelled. */
public static final String CANCELLED = "CANCELLED";
public static final int LOG_CONSUMER_BORDER_BYTES = 1024 * 1024;

/** First build info. */
@Nonnull private final BuildRef firstBuildInfo;

/** String Compactor. */
private IStringCompactor compactor;

/** Builds: Single execution. */
@@ -113,6 +124,23 @@ public boolean onlyCancelledBuilds() {
return buildsStream().flatMap(SingleBuildRunCtx::getProblemsStream);
}


/**
* @param problemCode Problem code.
* @return count of builds having particular problem type
*/
public long buildsCntHavingBuildProblem(String problemCode) {
int problemCodeId = compactor.getStringId(problemCode);
int cnt = 0;

for (SingleBuildRunCtx ctx : builds) {
if (ctx.hasBuildProblemType(problemCodeId))
cnt++;
}

return cnt;
}

public List<SingleBuildRunCtx> getBuilds() {
return builds;
}
@@ -155,7 +183,7 @@ public boolean hasJvmCrashProblem() {
}

public long getJvmCrashProblemCount() {
return buildsStream().filter(ISuiteResults::hasJvmCrashProblem).count();
return buildsCntHavingBuildProblem(ProblemOccurrence.TC_JVM_CRASH);
}

public boolean hasOomeProblem() {
@@ -264,9 +292,8 @@ public Stream<Map.Entry<String, Long>> getTopLogConsumers() {
(testName, logCheckResult) -> {
//todo may be it is better to find avg
long bytes = (long)logCheckResult.getLogSizeBytes();
if (bytes > 1024 * 1024) {
if (bytes > LOG_CONSUMER_BORDER_BYTES)
logSizeBytes.merge(testName, bytes, Math::max);
}
}
);
});
@@ -309,7 +336,7 @@ public int getBuildId() {
return firstBuildInfo.getId();
}

boolean isFailed() {
public boolean isFailed() {
return failedTests() != 0 || hasAnyBuildProblemExceptTestOrSnapshot() || onlyCancelledBuilds();
}

@@ -560,4 +587,44 @@ public StringBuilder appendCriticalFailRate(IRunHistory baseBranchHist, StringBu

return res;
}

public boolean hasCriticalProblem() {
return hasJvmCrashProblem()
|| hasTimeoutProblem()
|| hasCompilationProblem()
|| hasMetricProblem();
}

public Integer totalTests() {
return (int)buildsStream().mapToInt(SingleBuildRunCtx::totalNotMutedTests).average().orElse(0.0);
}

public Integer trustedTests(ITeamcityIgnited tcIgnited,
String normalizedBaseBranch) {

AtomicInteger trustedCnt = new AtomicInteger();
Map<Integer, TestCompactedMult> res = new HashMap<>();

//todo can cache mult occurrences in ctx
builds.forEach(singleBuildRunCtx -> {
saveToMap(res, singleBuildRunCtx.getAllTests()
.filter(t -> !t.isIgnoredTest() && !t.isMutedTest()));
});
Integer branchName = compactor.getStringIdIfPresent(normalizedBaseBranch);
Integer suiteName = compactor.getStringIdIfPresent( buildTypeId());

// res.clear(); //todo enable feature back

//todo can cache fail rate in mult occur
res.keySet().forEach((testNameId) -> {
IRunHistory stat = tcIgnited.getTestRunHist(testNameId, suiteName, branchName);
String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);
boolean b = testBlockerComment != null;
if (b) // this test will be considered as blocker if will fail
trustedCnt.addAndGet(1);
});

return trustedCnt.get();
}

}
@@ -32,20 +32,19 @@
import java.util.regex.Pattern;
import java.util.stream.Stream;
import javax.annotation.Nonnull;

import org.apache.ignite.tcignited.buildlog.ILogCheckResult;
import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
import org.apache.ignite.tcservice.ITeamcity;
import org.apache.ignite.tcbot.common.conf.IBuildParameterSpec;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.buildtype.ParametersCompacted;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.ci.util.FutureUtil;
import org.apache.ignite.tcbot.common.conf.IBuildParameterSpec;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.buildlog.ILogCheckResult;
import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
import org.apache.ignite.tcservice.ITeamcity;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
import org.jetbrains.annotations.Nullable;

/**
@@ -95,6 +94,7 @@ public Integer buildId() {
return getProblemsStream().anyMatch(p -> p.isCompilationError(compactor));
}


public boolean hasTimeoutProblem() {
return getExecutionTimeoutCount() > 0;
}
@@ -272,6 +272,9 @@ public void addTag(@Nullable String lb) {
this.tags.add(lb);
}

/**
* @return Build tags associated with this run. May be Java version, area of responibilty - any text mark.
*/
public Set<String> tags() {
return tags;
}
@@ -325,4 +328,21 @@ else if (ITeamcity.SUITE_NAME_PROPERTY.equals(parmKey))

return propVal;
}

public boolean hasBuildProblemType(int id) {
return buildCompacted.hasBuildProblemType(id);
}

public boolean hasSuiteIncompleteFailure() {
return hasJvmCrashProblem()
|| hasTimeoutProblem()
|| hasOomeProblem()
|| hasExitCodeProblem()
|| hasCompilationProblem()
|| hasMetricProblem();
}

public int totalNotMutedTests() {
return buildCompacted.totalNotMutedTests();
}
}
@@ -21,11 +21,15 @@
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.history.IRunHistSummary;
import org.apache.ignite.tcignited.history.IRunStat;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;

/**
* Test occurrence merged from several runs.
*/
public class TestCompactedMult implements IMultTestOccurrence {
private final List<TestCompacted> occurrences = new ArrayList<>();
private IStringCompactor compactor;
@@ -78,22 +82,29 @@ private int getFailedButNotMutedCount() {
.collect(Collectors.toList());
}

@Override public String getPossibleBlockerComment(IRunHistory baseBranchStat) {
if (baseBranchStat == null)
return "History for base branch is absent.";
/**
* @param baseBranchStat Base branch statistics.
* @return non null comment in case test failure is a blocker for merge into base branch.
*/
public static String getPossibleBlockerComment(IRunHistSummary baseBranchStat) {
if (baseBranchStat == null)
return "History for base branch is absent.";

String flakyComments = baseBranchStat.getFlakyComments();
boolean flaky = baseBranchStat.isFlaky();

boolean lowFailureRate = baseBranchStat.getFailRate() * 100.0f < 4.;
float failRate = baseBranchStat.getFailRate();
boolean lowFailureRate = failRate * 100.0f < 4.;

if (lowFailureRate && flakyComments == null) {
return "Test has low fail rate in base branch "
+ baseBranchStat.getFailPercentPrintable()
+ "% and is not flaky";
}
if (lowFailureRate && !flaky) {
String runStatPrintable = IRunStat.getPercentPrintable(failRate * 100.0f);

return null;
}
return "Test has low fail rate in base branch "
+ runStatPrintable
+ "% and is not flaky";
}

return null;
}

public void add(TestCompacted next) {
occurrences.add(next);
@@ -38,7 +38,7 @@
import org.apache.ignite.tcbot.persistence.IgniteStringCompactor;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildDao;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.history.RunHistCompactedDao;
import org.apache.ignite.ci.user.TcHelperUser;
import org.apache.ignite.tcservice.util.XmlUtil;
@@ -361,13 +361,14 @@ protected void fillBuildCounts(MultBuildRunCtx outCtx,
protected void analyzeTests(MultBuildRunCtx outCtx, ITeamcityIgnited teamcity,
ProcessLogsMode procLog) {
for (SingleBuildRunCtx ctx : outCtx.getBuilds()) {
if ((procLog == ProcessLogsMode.SUITE_NOT_COMPLETE && ctx.hasSuiteIncompleteFailure())
boolean incompleteFailure = ctx.hasSuiteIncompleteFailure();
if ((procLog == ProcessLogsMode.SUITE_NOT_COMPLETE && incompleteFailure)
|| procLog == ProcessLogsMode.ALL)
ctx.setLogCheckResFut(
CompletableFuture.supplyAsync(
() -> buildLogProcessor.analyzeBuildLog(teamcity,
ctx.buildId(),
ctx.hasSuiteIncompleteFailure()),
incompleteFailure),
tcUpdatePool.getService()));
}
}
@@ -23,6 +23,7 @@
import java.util.stream.Collectors;
import javax.inject.Inject;
import org.apache.ignite.ci.analysis.FullChainRunCtx;
import org.apache.ignite.ci.analysis.TestCompactedMult;
import org.apache.ignite.ci.analysis.mode.LatestRebuildMode;
import org.apache.ignite.ci.analysis.mode.ProcessLogsMode;
import org.apache.ignite.ci.github.ignited.IGitHubConnIgnited;
@@ -149,7 +150,7 @@ else if (FullQueryParams.CHAIN.equals(act))
runningUpdates.addAndGet(cnt0);

//fail rate reference is always default (master)
chainStatus.initFromContext(tcIgnited, ctx, baseBranch, compactor);
chainStatus.initFromContext(tcIgnited, ctx, baseBranch, compactor, false); // don't need for PR

chainStatus.initJiraAndGitInfo(ticketMatcher, jiraIntegration, gitHubConnIgnited);
}
@@ -185,8 +186,8 @@ public List<SuiteCurrentStatus> getBlockersSuitesStatuses(String buildTypeId, St

String baseBranch = ITeamcity.DEFAULT;

final FullChainRunCtx ctx = buildChainProcessor.loadFullChainContext(
tcIgnited,
FullChainRunCtx ctx = buildChainProcessor.loadFullChainContext(
tcIgnited,
hist,
LatestRebuildMode.LATEST,
ProcessLogsMode.SUITE_NOT_COMPLETE,
@@ -217,10 +218,10 @@ private List<SuiteCurrentStatus> findBlockerFailures(FullChainRunCtx fullChainRu

String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config());

List<TestFailure> failures = ctx.getFailedTests().stream().map(occurrence -> {
List<TestFailure> failures = ctx.getFailedTests().stream().map(occurrence -> {
IRunHistory stat = tcIgnited.getTestRunHist(occurrence.getName(), normalizedBaseBranch);

String testBlockerComment = occurrence.getPossibleBlockerComment(stat);
String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);

if (!Strings.isNullOrEmpty(testBlockerComment)) {
final TestFailure failure = new TestFailure();
@@ -239,7 +240,7 @@ private List<SuiteCurrentStatus> findBlockerFailures(FullChainRunCtx fullChainRu
SuiteCurrentStatus suiteUi = new SuiteCurrentStatus();
suiteUi.testFailures = failures;

suiteUi.initFromContext(tcIgnited, ctx, baseBranch, compactor, false);
suiteUi.initFromContext(tcIgnited, ctx, baseBranch, compactor, false, false);

return suiteUi;
}

0 comments on commit 912f8f5

Please sign in to comment.