Skip to content
Permalink
Browse files
Performance fixes:
- Invocation history for tests (parameters removed for now, Tree Map repaced by list to reduce Heap usage);
- Fat build test invocation v2.0 (no odd lines in test details, no bit set to reduce Heap);
- Reconciliation of build refs and fat build made per partition to collocate reads of fat builds partition files;

Fixes #142
  • Loading branch information
dspavlov committed Aug 3, 2019
1 parent 3f55641 commit 02adaa8499eaef39da3d792b885b1c3297f51d7a
Showing 38 changed files with 1,041 additions and 565 deletions.
@@ -277,13 +277,6 @@ private String showTestFailData(DsTestFailureUi testFail) {
if(isDefinedAndFilled(hist.recent))
testFailTitle = "recent rate: " + hist.recent.failures + " fails / " + hist.recent.runs + " runs" ;

if(isDefinedAndFilled(hist.allTime) && isDefinedAndFilled(hist.allTime.failures)) {
testFailTitle +=
"; all history: " + hist.allTime.failureRate + "% ["+
hist.allTime.failures + " fails / " +
hist.allTime.runs + " runs] " ;
}

histContent += " <span title='" +testFailTitle + "'>";

if (isDefinedAndFilled(hist.recent) && isDefinedAndFilled(hist.recent.failureRate))
@@ -624,10 +624,6 @@ function showSuiteData(suite, settings, prNum) {

moreInfoTxt += "Recent fails : "+ suite.failureRate +"% [" + suite.failures + " fails / " + suite.runs + " runs]; <br> " ;

if(isDefinedAndFilled(suite.failsAllHist) && isDefinedAndFilled(suite.failsAllHist.failures)) {
moreInfoTxt += "All hist fails: "+ suite.failsAllHist.failureRate +"% [" + suite.failsAllHist.failures + " fails / " + suite.failsAllHist.runs + " runs]; <br> " ;
}

if(isDefinedAndFilled(suite.criticalFails) && isDefinedAndFilled(suite.criticalFails.failures)) {
moreInfoTxt += "Critical recent fails: "+ suite.criticalFails.failureRate + "% [" + suite.criticalFails.failures + " fails / " + suite.criticalFails.runs + " runs]; <br> " ;
}
@@ -927,13 +923,6 @@ function showTestFailData(testFail, isFailureShown, settings) {
if(isDefinedAndFilled(hist.recent) && isDefinedAndFilled(hist.recent.failures))
testFailTitle = "recent rate: " + hist.recent.failures + " fails / " + hist.recent.runs + " runs" ;

if(isDefinedAndFilled(hist.allTime) && isDefinedAndFilled(hist.allTime.failures)) {
testFailTitle +=
"; all history: " + hist.allTime.failureRate + "% ["+
hist.allTime.failures + " fails / " +
hist.allTime.runs + " runs] " ;
}

histContent += " <span title='" +testFailTitle + "'>";

if (isDefinedAndFilled(hist.recent) && isDefinedAndFilled(hist.recent.failureRate))
@@ -34,7 +34,7 @@
import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.ci.teamcity.ignited.InMemoryStringCompactor;
import org.apache.ignite.tcbot.persistence.InMemoryStringCompactor;
import org.apache.ignite.tcignited.SyncMode;
import org.apache.ignite.ci.teamcity.ignited.TeamcityIgnitedMock;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
@@ -116,7 +116,7 @@ else if (test.getName().equals(TEST_FAILING_EVERY_TIME))
TestOccurrenceFull t1 = new TestOccurrenceFull();
t1.name = UNIQUE_FAILED_TEST + j;
t1.status = TestOccurrence.STATUS_SUCCESS;
pds1.addTests(c, Lists.newArrayList(t1));
pds1.addTests(c, Lists.newArrayList(t1), null);

builds.put(pds1.id(), pds1);
}
@@ -176,7 +176,7 @@ public void addTestBuild(IStringCompactor c, Map<Integer, FatBuildCompacted> bui
TestOccurrenceFull t1 = new TestOccurrenceFull();
t1.name = UNIQUE_FAILED_TEST + i;
t1.status = TestOccurrence.STATUS_FAILURE;
pds1.addTests(c, Lists.newArrayList(t1));
pds1.addTests(c, Lists.newArrayList(t1), null);

builds.put(pds1.id(), pds1);

@@ -189,7 +189,7 @@ public void addTestBuild(IStringCompactor c, Map<Integer, FatBuildCompacted> bui
TestOccurrenceFull t3 = new TestOccurrenceFull();
t3.name = TEST_FAILING_EVERY_TIME;
t3.status = TestOccurrence.STATUS_FAILURE;
pds2.addTests(c, Lists.newArrayList(t2, t3));
pds2.addTests(c, Lists.newArrayList(t2, t3), null);

builds.put(pds2.id(), pds2);
}
@@ -45,7 +45,7 @@
import org.apache.ignite.tcbot.notify.ISlackSender;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
import org.apache.ignite.ci.teamcity.ignited.InMemoryStringCompactor;
import org.apache.ignite.tcbot.persistence.InMemoryStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.TeamcityIgnitedProviderMock;
import org.apache.ignite.tcbot.common.conf.IDataSourcesConfigSupplier;
import org.apache.ignite.tcbot.common.conf.TcBotWorkDir;
@@ -46,8 +46,6 @@
import org.apache.ignite.ci.teamcity.ignited.TeamcityIgnitedProviderMock;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.user.ITcBotUserCreds;
import org.apache.ignite.tcbot.engine.ui.DsSuiteUi;
import org.apache.ignite.tcbot.engine.ui.DsTestFailureUi;
import org.jetbrains.annotations.NotNull;
import org.junit.Before;
import org.junit.Test;
@@ -210,7 +208,7 @@ public Map<Integer, FatBuildCompacted> initHistory(IStringCompactor c) {
createFailedTest(2L, TEST_WITH_HISTORY_FAILING_IN_MASTER),
createPassingTest(3L, TEST_WITH_HISTORY_PASSING_IN_MASTER),
createTest(50L, TEST_FLAKY_IN_MASTER, i % 2 == 0),
createPassingTest(400L, TEST_WAS_FIXED_IN_MASTER)));
createPassingTest(400L, TEST_WAS_FIXED_IN_MASTER)), null);

if (i % 7 == 1) {
ProblemOccurrence timeout = new ProblemOccurrence();
@@ -227,7 +225,7 @@ public Map<Integer, FatBuildCompacted> initHistory(IStringCompactor c) {
addBuildsToEmulatedStor(createFailedBuild(c, CACHE_1,
ITeamcity.DEFAULT, i, ageMs + (i * 10000))
.addTests(c, Lists.newArrayList(
createFailedTest(400L, TEST_WAS_FIXED_IN_MASTER))));
createFailedTest(400L, TEST_WAS_FIXED_IN_MASTER)), null));
}

for (int i = 0; i < 10; i++) {
@@ -239,7 +237,7 @@ public Map<Integer, FatBuildCompacted> initHistory(IStringCompactor c) {
createPassingTest(2L, TEST_WITH_HISTORY_FAILING_IN_MASTER),
createPassingTest(3L, TEST_WITH_HISTORY_PASSING_IN_MASTER),
createPassingTest(50L, TEST_FLAKY_IN_MASTER),
createPassingTest(400L, TEST_WAS_FIXED_IN_MASTER)));
createPassingTest(400L, TEST_WAS_FIXED_IN_MASTER)), null);

addBuildsToEmulatedStor(successfull);
}
@@ -254,7 +252,7 @@ public Map<Integer, FatBuildCompacted> initHistory(IStringCompactor c) {
.addTests(c,
Lists.newArrayList(
createTest(1L, TEST_RARE_FAILED_WITHOUT_CHANGES, !failNoChanges),
createTest(2L, TEST_RARE_FAILED_WITH_CHANGES, !failWithChanges)));
createTest(2L, TEST_RARE_FAILED_WITH_CHANGES, !failWithChanges)), null);

if (failWithChanges || i == 56) // add change to test status change after failure.
fatBuild.changes(new int[] {1000000 + i, 1000020 + i});
@@ -286,7 +284,7 @@ public void initBuildChain(IStringCompactor c, String btId, String branch) {
createFailedTest(2L, TEST_WITH_HISTORY_FAILING_IN_MASTER),
createFailedTest(3L, TEST_WITH_HISTORY_PASSING_IN_MASTER),
createFailedTest(50L, TEST_FLAKY_IN_MASTER),
createFailedTest(400L, TEST_WAS_FIXED_IN_MASTER)));
createFailedTest(400L, TEST_WAS_FIXED_IN_MASTER)), null);

cache1.snapshotDependencies(new int[] {buildBuild.id()});

@@ -319,7 +317,7 @@ public void initChainForFlakyTest(IStringCompactor c, String btId, String branch
.addTests(c,
Lists.newArrayList(
createFailedTest(1L, TEST_RARE_FAILED_WITH_CHANGES),
createFailedTest(2L, TEST_RARE_FAILED_WITHOUT_CHANGES)));
createFailedTest(2L, TEST_RARE_FAILED_WITHOUT_CHANGES)), null);

cache9.snapshotDependencies(new int[] {buildBuild.id()});

@@ -413,7 +411,7 @@ public void initChainForTemplateDetection(IStringCompactor c, String btId, Strin
createFailedBuild(c, CACHE_8, branch, 9331 + i, 100090)
.addTests(c,
Lists.newArrayList(
createTest(1L, TEST_BECAME_FAILED_IN_BRANCH, i < firstFailedBuild)))
createTest(1L, TEST_BECAME_FAILED_IN_BRANCH, i < firstFailedBuild)), null)
.snapshotDependencies(new int[] {buildBuild.id()});

if (i == firstFailedBuild)
@@ -107,7 +107,7 @@ public void testTrackedBranchChainsProcessor() {
.addTests(c,
Lists.newArrayList(
createTest(1L, TEST_RARE_FAILED_WITHOUT_CHANGES, false),
createTest(2L, TEST_RARE_FAILED_WITH_CHANGES, false)));
createTest(2L, TEST_RARE_FAILED_WITH_CHANGES, false)), null);

fatBuild.changes(new int[] {1000000 + buildCnt, 1000020 + buildCnt});

@@ -150,14 +150,14 @@ public void emulateHistory(String chainId,
for (int i = 0; i < histLen; i++) {
FatBuildCompacted pds1Build
= createFatBuild(c, "PDS1", ITeamcity.DEFAULT, 1100 + i, 1000 * i, false)
.addTests(c, testsMapToXmlModel(pds1Hist, histLen, i))
.addTests(c, testsMapToXmlModel(pds1Hist, histLen, i), null)
.changes(new int[] {i});

apacheBuilds.put(pds1Build.id(), pds1Build);

FatBuildCompacted pds2Build
= createFatBuild(c, "PDS2_noChanges", ITeamcity.DEFAULT, 1200 + i, 1000 * i, false)
.addTests(c, testsMapToXmlModel(buildWoChanges, histLen, i));
.addTests(c, testsMapToXmlModel(buildWoChanges, histLen, i), null);

apacheBuilds.put(pds2Build.id(), pds2Build);

@@ -31,6 +31,7 @@
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.SyncMode;
import org.apache.ignite.tcignited.build.TestCompactedV2;
import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.jetbrains.annotations.NotNull;
@@ -177,9 +178,9 @@ public static void initHistory(IStringCompactor c, Map<RunHistKey, RunHistCompac

final RunHistCompacted hist = histCache.computeIfAbsent(histKey, RunHistCompacted::new);

Invocation inv = testCompacted.toInvocation(build, (k, v) -> true, successStatusStrId);
Invocation inv = TestCompactedV2.toInvocation(testCompacted, build, (k, v) -> true, successStatusStrId);

hist.innerAddInvocation(inv);
hist.addInvocation(inv);
});
}

@@ -32,6 +32,6 @@ dependencies {
//impl is used by TC module, only converters needed here
compile group: 'javax.xml.bind', name: 'jaxb-api', version: '2.3.1'

testCompile group: 'junit', name: 'junit', version: junitVer
testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoVer
testCompile group: 'junit', name: 'junit', version: junitVer;
testCompile group: 'org.mockito', name: 'mockito-core', version: mockitoVer;
}
@@ -39,13 +39,13 @@
import javax.annotation.Nullable;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.tcbot.common.util.CollectionUtil;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.build.ITest;
import org.apache.ignite.tcignited.buildlog.ILogCheckResult;
import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
import org.apache.ignite.tcignited.history.IRunHistory;
@@ -334,7 +334,7 @@ public List<TestCompactedMult> getFilteredTests(Predicate<TestCompactedMult> fil
.collect(Collectors.toList());
}

public void saveToMap(Map<Integer, TestCompactedMult> res, Stream<TestCompacted> tests) {
public void saveToMap(Map<Integer, TestCompactedMult> res, Stream<ITest> tests) {
tests.forEach(testCompacted -> {
res.computeIfAbsent(testCompacted.testName(), k -> new TestCompactedMult(compactor, this))
.add(testCompacted);
@@ -38,15 +38,14 @@
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.ProblemCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.tcbot.common.conf.IBuildParameterSpec;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.common.util.FutureUtil;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.build.ITest;
import org.apache.ignite.tcignited.buildlog.ILogCheckResult;
import org.apache.ignite.tcignited.buildlog.ITestLogCheckResult;
import org.apache.ignite.tcservice.ITeamcity;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;

/**
* Single build occurrence with changes, tag and log results.
@@ -196,15 +195,11 @@ public Stream<String> getFailedNotMutedTestNames() {
return isComposite() ? Stream.empty() : buildCompacted.getFailedNotMutedTestNames(compactor);
}

public Stream<TestCompacted> getFailedNotMutedTests() {
return isComposite() ? Stream.empty() : buildCompacted.getFailedNotMutedTests(compactor);
}

public Stream<String> getAllTestNames() {
return buildCompacted.getAllTestNames(compactor);
}

public Stream<TestCompacted> getAllTests() {
public Stream<ITest> getAllTests() {
return isComposite() ? Stream.empty() : buildCompacted.getAllTests();
}

@@ -21,25 +21,23 @@
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.Nullable;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.TestCompacted;
import org.apache.ignite.tcbot.common.TcBotConst;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.build.ITest;
import org.apache.ignite.tcignited.history.IRunHistSummary;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcignited.history.IRunStat;
import org.apache.ignite.tcignited.history.ISuiteRunHistory;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrenceFull;

/**
* Test occurrence merged from several runs.
*/
public class TestCompactedMult {
private final List<TestCompacted> occurrences = new ArrayList<>();
private final List<ITest> occurrences = new ArrayList<>();
private IStringCompactor compactor;
private MultBuildRunCtx ctx;
private long avgDuration = -1;
@@ -65,7 +63,7 @@ public String getName() {
}

public boolean isInvestigated() {
return occurrences.stream().anyMatch(TestCompacted::isInvestigated);
return occurrences.stream().anyMatch(ITest::isInvestigated);
}

/** */
@@ -82,7 +80,7 @@ public int failuresCount() {
public long getAvgDurationMs() {
if (avgDuration < 0) {
avgDuration = (long)occurrences.stream()
.map(TestCompacted::getDuration)
.map(ITest::getDuration)
.filter(Objects::nonNull)
.mapToInt(i -> i)
.average()
@@ -95,7 +93,7 @@ public long getAvgDurationMs() {
/**
*
*/
public Stream<TestCompacted> getInvocationsStream() {
public Stream<ITest> getInvocationsStream() {
return occurrences.stream();
}

@@ -135,7 +133,7 @@ public String getPossibleBlockerComment(IRunHistSummary baseBranchStat) {
return null;
}

public void add(TestCompacted next) {
public void add(ITest next) {
occurrences.add(next);
}

@@ -162,7 +160,7 @@ public boolean isFailedButNotMuted() {
*
*/
public boolean isMutedOrIgored() {
return occurrences.stream().anyMatch(TestCompacted::isMutedOrIgnored);
return occurrences.stream().anyMatch(ITest::isMutedOrIgnored);
}

/**
@@ -82,9 +82,6 @@ public class DsSuiteUi extends ShortSuiteUi {
/** Branch name in teamcity identification. */
public String branchName;

/** Failure summary in tracked branch according to all runs history. */
@Nonnull public DsHistoryStatUi failsAllHist = new DsHistoryStatUi();

/** Failure summary in tracked branch according to all runs history. */
@Nonnull public DsHistoryStatUi criticalFails = new DsHistoryStatUi();

@@ -274,10 +271,6 @@ private IRunHistory initSuiteStat(ITeamcityIgnited tcIgnited,
criticalFails.runs = runs;
criticalFails.failureRate = statInBaseBranch.getCriticalFailPercentPrintable();

failsAllHist.failures = statInBaseBranch.getFailuresAllHist();
failsAllHist.runs = statInBaseBranch.getRunsAllHist();
failsAllHist.failureRate = statInBaseBranch.getFailPercentAllHistPrintable();

latestRuns = statInBaseBranch.getLatestRunResults();
}

@@ -391,7 +384,6 @@ private static String buildWebLinkToHist(ITeamcityIgnited teamcity, MultBuildRun
Objects.equals(serverId, status.serverId) &&
Objects.equals(suiteId, status.suiteId) &&
Objects.equals(branchName, status.branchName) &&
Objects.equals(failsAllHist, status.failsAllHist) &&
Objects.equals(criticalFails, status.criticalFails) &&
Objects.equals(latestRuns, status.latestRuns) &&
Objects.equals(userCommits, status.userCommits) &&
@@ -411,7 +403,7 @@ private static String buildWebLinkToHist(ITeamcityIgnited teamcity, MultBuildRun
@Override public int hashCode() {
return Objects.hash(super.hashCode(), name, result, hasCriticalProblem, webToHist,
webToHistBaseBranch, webToBuild, testFailures, topLongRunning, warnOnly, logConsumers, webUrlThreadDump,
runningBuildCount, queuedBuildCount, serverId, suiteId, branchName, failsAllHist, criticalFails, latestRuns,
runningBuildCount, queuedBuildCount, serverId, suiteId, branchName, criticalFails, latestRuns,
userCommits, failedTests, durationPrintable, durationNetTimePrintable, sourceUpdateDurationPrintable,
artifcactPublishingDurationPrintable, dependeciesResolvingDurationPrintable, testsDurationPrintable,
lostInTimeouts, problemRef, blockerComment);

0 comments on commit 02adaa8

Please sign in to comment.