Skip to content
Permalink
Browse files
IGNITE-10095: Filter suite and test history by build tag finished, Su…
…ite History made versioned entity and enforced to rebuild for tags filter, UI bugs fixed - Fixes #144.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
  • Loading branch information
dspavlov committed Aug 5, 2019
1 parent 6d7b17d commit 09bb8018cdf9226718922b5ee393e9f6157c0e1d
Showing 32 changed files with 355 additions and 264 deletions.
@@ -67,7 +67,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;
import static org.apache.ignite.tcignited.buildref.BranchEquivalence.normalizeBranch;

/**
*
@@ -28,7 +28,7 @@
public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot";

/** TC Bot Version. */
public static final String VERSION = "20190803";
public static final String VERSION = "20190804";

/** Java version, where Web App is running. */
public String javaVer;
@@ -92,7 +92,7 @@ public void showChainOnServersResults(StringBuilder res, Integer buildId, String
status.chainName = ctx.suiteName();

IStringCompactor c = injector.getInstance(IStringCompactor.class);
status.initFromContext(tcIgn, ctx, failRateBranch, c, false, null, null, -1);
status.initFromContext(tcIgn, ctx, failRateBranch, c, false, null, null, -1, null);

res.append(showChainAtServerData(status));

@@ -141,7 +141,7 @@ public DsSummaryUi getBuildTestFailsNoSync(
if (cnt > 0)
runningUpdates.addAndGet(cnt);

chainStatus.initFromContext(tcIgnited, ctx, failRateBranch, injector.getInstance(IStringCompactor.class), false, null, null, -1);
chainStatus.initFromContext(tcIgnited, ctx, failRateBranch, injector.getInstance(IStringCompactor.class), false, null, null, -1, null);

res.addChainOnServer(chainStatus);

@@ -110,7 +110,11 @@
gVue.$data.count = count;
}

gVue.$data.trustedTests = findGetParameter("trustedTests") === "true";
let parm = findGetParameter("trustedTests");
if (isDefinedAndFilled(parm)) {
gVue.$data.trustedTests = parm === "true";
}

gVue.$data.checkAllLogs = findGetParameter("checkAllLogs") === "true";
gVue.$data.hideFlakyFailures = findGetParameter("hideFlakyFailures") === "true";

@@ -267,7 +271,7 @@

function showData(result) {
let setOfTags = new Set(gVue.$data.tagsPresent);
let setOfHistTags = new Set(gVue.$data.tagForHistSelected);
let setOfHistTags = new Set(gVue.$data.tagsForHistPresent);
for (let i = 0; i < result.servers.length; i++) {
let chain = result.servers[i];

@@ -320,7 +324,7 @@
</option>
</select>

<span title="Tag filter for filtering history of suites">History tag filter: </span>
&nbsp;&nbsp;&nbsp;<span title="Tag filter for filtering history of suites">History tag filter: </span>
<select v-model="tagForHistSelected" @change="formChanged">
<option disabled value="">Please select one</option>

@@ -334,20 +338,20 @@
<option value="Failures">Show failures only</option>
<option value="AllSuites">Show all suites</option>
</select>
<span>Sort: </span>
&nbsp;&nbsp;&nbsp;<span>Sort: </span>
<select v-model="sortOption" @change="formChanged">
<option value="FailureRate">Failure Rate</option>
<option value="SuiteDuration">Suite Duration</option>
</select>
<span>Merge Builds: </span>
&nbsp;&nbsp;&nbsp;<span>Merge Builds: </span>
<input v-model.number="count" type="number" @change="formChanged">

<br>&nbsp;&nbsp;<span title="Show trusted tests count">Trusted tests: </span>
<br><span title="Show trusted tests count">Trusted tests: </span>
<input type="checkbox" v-model="trustedTests" @change="formChanged">
&nbsp;&nbsp;<span title="Download and parse all logs">Check logs: </span>
&nbsp;&nbsp;&nbsp;<span title="Download and parse all logs">Check logs: </span>
<input type="checkbox" v-model="checkAllLogs" @change="formChanged">

<span>Show test longer than, seconds: </span>
&nbsp;&nbsp;&nbsp;<span>Show test longer than, seconds: </span>
<input v-model.number="showTestLongerThan" type="number" @change="formChanged">
</td>

@@ -183,7 +183,7 @@ public static void initHistory(IStringCompactor c, Map<RunHistKey, RunHistCompac

final RunHistCompacted hist = histCache.computeIfAbsent(histKey, RunHistCompacted::new);

Invocation inv = TestCompactedV2.toInvocation(testCompacted, build, (k, v) -> false, successStatusStrId);
Invocation inv = TestCompactedV2.toInvocation(testCompacted, build, successStatusStrId);

hist.addInvocation(inv);
});
@@ -50,7 +50,6 @@
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcignited.history.BuildStartTimeStorage;
import org.apache.ignite.tcignited.history.RunHistSync;
import org.apache.ignite.tcservice.ITeamcity;
import org.apache.ignite.tcservice.TeamcityServiceConnection;
import org.apache.ignite.tcservice.http.ITeamcityHttpConnection;
@@ -623,8 +622,6 @@ public void testHistoryBackgroundUpdateWorks() {
buildRefDao.save(srvIdMaskHigh, new BuildRefCompacted(build));
});

final RunHistSync histSync = injector.getInstance(RunHistSync.class);

final IRunHistory testRunHist = srv.getTestRunHist(c.getStringId(PrChainsProcessorTest.TEST_FLAKY_IN_MASTER),
c.getStringId(PrChainsProcessorTest.CACHE_1),
c.getStringId(branch));
@@ -41,6 +41,7 @@
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.ci.teamcity.ignited.buildtype.ParametersCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.Invocation;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.util.FutureUtil;
import org.apache.ignite.tcbot.engine.pool.TcUpdatePool;
@@ -50,8 +51,8 @@
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.SyncMode;
import org.apache.ignite.tcignited.buildlog.IBuildLogProcessor;
import org.apache.ignite.tcignited.buildref.BranchEquivalence;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.RunHistSync;
import org.apache.ignite.tcservice.model.hist.BuildRef;
import org.apache.ignite.tcservice.model.result.Build;
import org.slf4j.Logger;
@@ -138,7 +139,7 @@ public List<LrTestsSuiteSummaryUi> loadLongRunningTestsSummary(
* @param failRateBranch Fail rate branch.
* @param mode background data update mode.
* @param sortOption how to sort suites in context, default is by failure rate (most often - first).
* @param requireParamVal Require exact parameters value presence in the build.
* @param requireParamVal Require exact parameters value presence in the build. Null means no filtering.
*/
@AutoProfiling
public FullChainRunCtx loadFullChainContext(
@@ -155,7 +156,7 @@ public FullChainRunCtx loadFullChainContext(
if (entryPoints.isEmpty())
return new FullChainRunCtx(Build.createFakeStub());

Integer failRateBranchId = compactor.getStringIdIfPresent(RunHistSync.normalizeBranch(failRateBranch));
Integer failRateBranchId = compactor.getStringIdIfPresent(BranchEquivalence.normalizeBranch(failRateBranch));

Map<Integer, Future<FatBuildCompacted>> builds = loadAllBuildsInChains(entryPoints, mode, tcIgn);

@@ -193,7 +194,7 @@ public FullChainRunCtx loadFullChainContext(

//ask for history for the suite in parallel
tcUpdatePool.getService().submit(() -> {
ctx.history(tcIgn, failRateBranchId);
ctx.history(tcIgn, failRateBranchId, null);
});

analyzeTests(ctx, tcIgn, procLog);
@@ -212,7 +213,7 @@ public FullChainRunCtx loadFullChainContext(

if (sortOption == null || sortOption == SortOption.FailureRate) {
function = ctx -> {
IRunHistory runStat = ctx.history(tcIgn, failRateBranchId);
IRunHistory runStat = ctx.history(tcIgn, failRateBranchId, null);

if (runStat == null)
return 0d;
@@ -398,19 +399,7 @@ else if (includeLatestRebuild == LatestRebuildMode.ALL)
private boolean hasAnyParameterValue(@Nonnull Map<Integer, Integer> requireParamVal, FatBuildCompacted fatBuild) {
ParametersCompacted parameters = fatBuild.parameters();

if (parameters == null)
return false;

Set<Map.Entry<Integer, Integer>> entries = requireParamVal.entrySet();
for (Map.Entry<Integer, Integer> next : entries) {
Integer key = next.getKey();

int valId = parameters.findPropertyStringId(key);
if (Objects.equals(next.getValue(), valId))
return true;
}

return false;
return Invocation.hasAnyParameterValue(parameters, requireParamVal);
}

@SuppressWarnings("WeakerAccess")
@@ -162,12 +162,12 @@ public List<SingleBuildRunCtx> getBuilds() {

/** */
public long getMetricProblemCount() {
return buildsStream().filter(ISuiteResults::hasMetricProblem).count();
return buildsStream().filter(SingleBuildRunCtx::hasMetricProblem).count();
}

/** */
public long getBuildMessageProblemCount() {
return buildsStream().filter(ISuiteResults::hasBuildMessageProblem).count();
return buildsStream().filter(SingleBuildRunCtx::hasBuildMessageProblem).count();
}

/** {@inheritDoc} */
@@ -177,7 +177,7 @@ public long getBuildMessageProblemCount() {

/** */
public long getCompilationProblemCount() {
return buildsStream().filter(ISuiteResults::hasCompilationProblem).count();
return buildsStream().filter(SingleBuildRunCtx::hasCompilationProblem).count();
}

/** {@inheritDoc} */
@@ -209,11 +209,11 @@ public boolean hasOomeProblem() {
}

private long getExitCodeProblemsCount() {
return buildsStream().filter(ISuiteResults::hasExitCodeProblem).count();
return buildsStream().filter(SingleBuildRunCtx::hasExitCodeProblem).count();
}

private long getOomeProblemCount() {
return buildsStream().filter(ISuiteResults::hasOomeProblem).count();
return buildsStream().filter(SingleBuildRunCtx::hasOomeProblem).count();
}

public int failedTests() {
@@ -672,19 +672,23 @@ public Optional<SingleBuildRunCtx> firstBuild() {
* @param tcIgn Tc ign.
* @param baseBranchId Base branch id.
*/
public IRunHistory history(ITeamcityIgnited tcIgn, Integer baseBranchId) {
public IRunHistory history(ITeamcityIgnited tcIgn,
@Nullable Integer baseBranchId,
@Nullable Map<Integer, Integer> requireParameters) {
if (baseBranchId == null)
return null;

ISuiteRunHistory suiteHist = suiteHist(tcIgn, baseBranchId);
ISuiteRunHistory suiteHist = suiteHist(tcIgn, baseBranchId, requireParameters);
if (suiteHist == null)
return null;

return suiteHist.self();
}

@Nullable
ISuiteRunHistory suiteHist(ITeamcityIgnited tcIgn, @Nullable Integer baseBranchId) {
ISuiteRunHistory suiteHist(ITeamcityIgnited tcIgn,
@Nullable Integer baseBranchId,
@Nullable Map<Integer, Integer> requireParameters) {
Integer buildTypeIdId = buildTypeIdId();
Preconditions.checkNotNull(buildTypeIdId, "Build type ID should be filled");

@@ -693,14 +697,17 @@ ISuiteRunHistory suiteHist(ITeamcityIgnited tcIgn, @Nullable Integer baseBranchI
return null;

try {
return histCacheMap.get(baseBranchId,
() -> {
return Optional.ofNullable(tcIgn.getSuiteRunHist(buildTypeIdId, baseBranchId));
})
ISuiteRunHistory suiteRunHistory = histCacheMap.get(baseBranchId,
() -> Optional.ofNullable(tcIgn.getSuiteRunHist(buildTypeIdId, baseBranchId)))
.orElse(null);

if (suiteRunHistory != null && requireParameters != null && !requireParameters.isEmpty())
return suiteRunHistory.filter(requireParameters);

return suiteRunHistory;
}
catch (ExecutionException e) {
throw ExceptionUtil.propagateException(e);
throw ExceptionUtil.propagateException(e);
}
}

@@ -17,12 +17,6 @@

package org.apache.ignite.tcbot.engine.chain;

import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
@@ -33,6 +27,14 @@
import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;

import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;

/**
* Test occurrence merged from several runs.
*/
@@ -138,11 +140,17 @@ public void add(ITest next) {
}

public IRunHistory history(ITeamcityIgnited ignited, @Nullable Integer baseBranchId) {
return history(ignited, baseBranchId, null);
}

public IRunHistory history(ITeamcityIgnited ignited,
@Nullable Integer baseBranchId,
@Nullable Map<Integer, Integer> requireParameters) {
Integer name = testName();
if (name == null || baseBranchId == null)
return null;

ISuiteRunHistory suiteRunHist = ctx.suiteHist(ignited, baseBranchId);
ISuiteRunHistory suiteRunHist = ctx.suiteHist(ignited, baseBranchId, requireParameters);

if (suiteRunHist == null)
return null;
@@ -49,9 +49,9 @@
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
import org.apache.ignite.tcignited.SyncMode;
import org.apache.ignite.tcignited.buildref.BranchEquivalence;
import org.apache.ignite.tcignited.creds.ICredentialsProv;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.RunHistSync;
import org.apache.ignite.tcservice.ITeamcity;

/**
@@ -165,7 +165,8 @@ else if (Action.CHAIN.equals(act))
runningUpdates.addAndGet(cnt0);

//fail rate reference is always default (master)
chainStatus.initFromContext(tcIgnited, ctx, baseBranchForTc, compactor, false, null, null, -1); // don't need for PR
chainStatus.initFromContext(tcIgnited, ctx, baseBranchForTc, compactor, false,
null, null, -1, null); // don't need for PR

initJiraAndGitInfo(chainStatus, jiraIntegration, gitHubConnIgnited);
}
@@ -293,7 +294,7 @@ public List<ShortSuiteUi> getBlockersSuitesStatuses(
private List<ShortSuiteUi> findBlockerFailures(FullChainRunCtx fullChainRunCtx,
ITeamcityIgnited tcIgnited,
String baseBranch) {
String normalizedBaseBranch = RunHistSync.normalizeBranch(baseBranch);
String normalizedBaseBranch = BranchEquivalence.normalizeBranch(baseBranch);
Integer baseBranchId = compactor.getStringIdIfPresent(normalizedBaseBranch);

Predicate<MultBuildRunCtx> filter = suite ->
@@ -302,7 +303,7 @@ private List<ShortSuiteUi> findBlockerFailures(FullChainRunCtx fullChainRunCtx,
return fullChainRunCtx
.filteredChildSuites(filter)
.map((ctx) -> {
IRunHistory statInBaseBranch = ctx.history(tcIgnited, baseBranchId);
IRunHistory statInBaseBranch = ctx.history(tcIgnited, baseBranchId, null);

String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config());

@@ -158,7 +158,8 @@ public class TrackedBranchChainsProcessor implements IDetailedStatusForTrackedBr
if (cnt > 0)
runningUpdates.addAndGet(cnt);

chainStatus.initFromContext(tcIgnited, ctx, baseBranchTc, compactor, calcTrustedTests, tagSelected, displayMode, maxDurationSec);
chainStatus.initFromContext(tcIgnited, ctx, baseBranchTc, compactor, calcTrustedTests, tagSelected,
displayMode, maxDurationSec, requireParamVal);

return chainStatus;
})

0 comments on commit 09bb801

Please sign in to comment.