Skip to content
Permalink
Browse files
Removed tests from other branches in Visa's new tests list. Fixes #183
Signed-off-by: Slava Koptilin <slava.koptilin@gmail.com>
  • Loading branch information
sergeyuttsel authored and sk0x50 committed Jan 12, 2021
1 parent f413e0a commit 728d98c2112479a171e9e104f55dd61598f88125
Showing 8 changed files with 211 additions and 17 deletions.
@@ -19,9 +19,7 @@

import com.google.inject.AbstractModule;
import com.google.inject.internal.SingletonScope;
import java.io.File;
import java.util.Properties;
import org.apache.ignite.ci.HelperConfig;
import org.apache.ignite.Ignite;
import org.apache.ignite.ci.github.PullRequest;
import org.apache.ignite.githubignited.IGitHubConnIgnited;
import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider;
@@ -42,6 +40,7 @@
import org.apache.ignite.tcbot.engine.conf.NotificationsConfig;
import org.apache.ignite.tcbot.engine.conf.TcServerConfig;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage;
import org.apache.ignite.tcbot.engine.user.IUserStorage;
import org.apache.ignite.tcbot.notify.IEmailSender;
import org.apache.ignite.tcbot.notify.ISlackSender;
@@ -50,12 +49,9 @@
import org.apache.ignite.tcbot.persistence.InMemoryStringCompactor;
import org.apache.ignite.ci.teamcity.ignited.TeamcityIgnitedProviderMock;
import org.apache.ignite.tcbot.common.conf.IDataSourcesConfigSupplier;
import org.apache.ignite.tcbot.common.conf.TcBotWorkDir;
import org.apache.ignite.tcignited.buildlog.IBuildLogProcessor;
import org.apache.ignite.tcservice.ITeamcityConn;
import org.mockito.Mockito;

import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
@@ -140,6 +136,9 @@ public MockBasedTcBotModule() {
bind(IEmailSender.class).toInstance(Mockito.mock(IEmailSender.class));
bind(ISlackSender.class).toInstance(Mockito.mock(ISlackSender.class));

bind(Ignite.class).toInstance(Mockito.mock(Ignite.class));
bind(NewTestsStorage.class).toInstance(Mockito.mock(NewTestsStorage.class));

super.configure();
}

@@ -26,6 +26,7 @@
import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
import org.apache.ignite.tcbot.engine.issue.IssuesStorage;
import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage;
import org.apache.ignite.tcbot.engine.tracked.IDetailedStatusForTrackedBranch;
import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor;
import org.apache.ignite.tcbot.engine.user.IUserStorage;
@@ -49,6 +50,8 @@ public class TcBotEngineModule extends AbstractModule {

bind(IUserStorage.class).to(UserAndSessionsStorage.class).in(new SingletonScope());

bind(NewTestsStorage.class).in(new SingletonScope());

install(new TcBotCommonModule());
}
}
}
@@ -26,7 +26,6 @@
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import javax.inject.Inject;
import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildConditionDao;
import org.apache.ignite.lang.IgniteBiTuple;
@@ -36,6 +35,7 @@
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.defect.DefectsStorage;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.buildlog.BuildLogCheckResultDao;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
@@ -48,7 +48,6 @@
import static java.util.stream.Collectors.mapping;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import static org.apache.ignite.tcignited.build.FatBuildDao.cacheKeyToSrvIdAndBuildId;

public class Cleaner {
private final AtomicBoolean init = new AtomicBoolean();
@@ -61,6 +60,7 @@ public class Cleaner {
@Inject private BuildStartTimeStorage buildStartTimeStorage;
@Inject private BuildConditionDao buildConditionDao;
@Inject private DefectsStorage defectsStorage;
@Inject private NewTestsStorage newTestsStorage;
@Inject private ITcBotConfig cfg;

/** Logger. */
@@ -132,6 +132,8 @@ private int removeCacheEntries(ZonedDateTime thresholdDate, int numOfItemsToDel)
defectsStorage.removeOldDefects(thresholdDate.minusDays(60).toInstant().toEpochMilli(), numOfItemsToDel);
issuesStorage.removeOldIssues(thresholdDate.minusDays(60).toInstant().toEpochMilli(), numOfItemsToDel);

newTestsStorage.removeOldTests(ZonedDateTime.now().minusDays(5).toInstant().toEpochMilli());

return oldBuildsKeys.size();
}

@@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ignite.tcbot.engine.newtests;

/** */
public class NewTestInfo {
/** */
private String branch;

/** */
private long timestamp;

/** */
public NewTestInfo(String branch, long timestamp) {
this.branch = branch;
this.timestamp = timestamp;
}

/** */
public String branch() {
return branch;
}

/** */
public long timestamp() {
return timestamp;
}
}
@@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ignite.tcbot.engine.newtests;

/** */
public class NewTestKey {
/** */
String srvId;

/** */
Long testId;

/** */
String baseBranch;

public NewTestKey(String srvId, Long testId, String baseBranch) {
this.srvId = srvId;
this.testId = testId;
this.baseBranch = baseBranch;
}
}
@@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.ignite.tcbot.engine.newtests;

import javax.cache.Cache;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.tcbot.persistence.CacheConfigs;

/**
* The storage contains tests which were identified as new tests in the tcbot visa
*/
public class NewTestsStorage {
/** */
@Inject
private Provider<Ignite> igniteProvider;

/** */
private IgniteCache<NewTestKey, NewTestInfo> cache() {
return botNewTestsCache(getIgnite());
}

/** */
private Ignite getIgnite() {
return igniteProvider.get();
}

/** */
public static IgniteCache<NewTestKey, NewTestInfo> botNewTestsCache(Ignite ignite) {
CacheConfiguration<NewTestKey, NewTestInfo> ccfg = CacheConfigs.getCache8PartsConfig("newTestsCache");

return ignite.getOrCreateCache(ccfg);
}

/** */
public boolean isNewTest(String srvId, Long testId, String baseBranch, String branch) {
NewTestInfo savedTest = cache().get(new NewTestKey(srvId, testId, baseBranch));

if (savedTest == null)
return true;
else
return savedTest.branch().startsWith(branch);
}

/** */
public boolean isNewTestAndPut(String srvId, Long testId, String baseBranch, String branch) {
NewTestKey testKey = new NewTestKey(srvId, testId, baseBranch);

NewTestInfo savedTest = cache().get(testKey);

if (savedTest == null) {
cache().put(testKey, new NewTestInfo(branch, System.currentTimeMillis()));
return true;
}
else
return savedTest.branch().startsWith(branch);
}

/** */
public void removeOldTests(long thresholdDate) {
ScanQuery<NewTestKey, NewTestInfo> scan =
new ScanQuery<>((key, testInfo) -> testInfo.timestamp() < thresholdDate);

for (Cache.Entry<NewTestKey, NewTestInfo> entry : cache().query(scan))
cache().remove(entry.getKey());
}
}
@@ -17,6 +17,7 @@
package org.apache.ignite.tcbot.engine.pr;

import com.google.common.base.Strings;
import java.time.LocalDate;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -27,7 +28,6 @@
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.ci.github.PullRequest;
import org.apache.ignite.githubignited.IGitHubConnIgnited;
import org.apache.ignite.githubignited.IGitHubConnIgnitedProvider;
@@ -44,6 +44,7 @@
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.conf.ITrackedBranch;
import org.apache.ignite.tcbot.engine.conf.ITrackedChain;
import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage;
import org.apache.ignite.tcbot.engine.ui.DsChainUi;
import org.apache.ignite.tcbot.engine.ui.DsSummaryUi;
import org.apache.ignite.tcbot.engine.ui.ShortSuiteUi;
@@ -59,8 +60,6 @@
import org.apache.ignite.tcignited.creds.ICredentialsProv;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcservice.ITeamcity;
import org.apache.ignite.tcservice.ITeamcityConn;
import org.apache.ignite.tcservice.TeamcityServiceConnection;

/**
* Process pull request/untracked branch chain at particular server.
@@ -95,6 +94,8 @@ private static class Action {

@Inject private UpdateCountersStorage countersStorage;

@Inject private NewTestsStorage newTestsStorage;

/**
* @param creds Credentials.
* @param srvCodeOrAlias Server code or alias.
@@ -174,7 +175,7 @@ else if (Action.CHAIN.equals(act))
//fail rate reference is always default (master)
chainStatus.initFromContext(tcIgnited, ctx, baseBranchForTc, compactor, false,
null, null, -1, null, false, false); // don't need for PR
chainStatus.findNewTests(ctx, tcIgnited, baseBranchForTc, compactor);
chainStatus.findNewTests(ctx, tcIgnited, baseBranchForTc, compactor, newTestsStorage);
initJiraAndGitInfo(chainStatus, jiraIntegration, gitHubConnIgnited);
}

@@ -393,7 +394,17 @@ private List<ShortSuiteNewTestsUi> findNewTests(FullChainRunCtx fullChainRunCtx,
.map((ctx) -> {
List<ShortTestUi> missingTests = ctx.getFilteredTests(test -> {
IRunHistory history = test.history(tcIgnited, baseBranchId, null);
return history == null && !test.isMutedOrIgored();
if (history == null && !test.isMutedOrIgored()) {

if (test.getId() != null &&
newTestsStorage.isNewTestAndPut(tcIgnited.serverCode(),
test.getId(), normalizedBaseBranch, ctx.branchName()))
return true;
else
return false;
}
else
return false;
})
.stream()
.map(occurrence -> new ShortTestUi().initFrom(occurrence, occurrence.isPassed()))
@@ -18,6 +18,7 @@
package org.apache.ignite.tcbot.engine.ui;

import com.google.common.base.Strings;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
@@ -32,11 +33,11 @@
import org.apache.ignite.tcbot.engine.chain.FullChainRunCtx;
import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx;
import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
import org.apache.ignite.tcbot.engine.newtests.NewTestsStorage;
import org.apache.ignite.tcbot.engine.tracked.DisplayMode;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.history.IRunHistory;
import org.apache.ignite.tcservice.ITeamcityConn;
import org.apache.ignite.tcservice.model.conf.BuildType;

import static org.apache.ignite.tcbot.engine.ui.DsSuiteUi.createOccurForLogConsumer;
@@ -294,15 +295,28 @@ public void initFromContext(ITeamcityIgnited tcIgnited,
public void findNewTests(FullChainRunCtx ctx,
ITeamcityIgnited tcIgnited,
String baseBranchTc,
IStringCompactor compactor){
IStringCompactor compactor,
NewTestsStorage newTestsStorage){
String failRateNormalizedBranch = normalizeBranch(baseBranchTc);
Integer baseBranchId = compactor.getStringIdIfPresent(failRateNormalizedBranch);

newTestsUi = ctx
.suites()
.map((suite) -> {
List<ShortTestUi> missingTests = suite.getFilteredTests(test -> {
IRunHistory history = test.history(tcIgnited, baseBranchId, null);
return history == null && !test.isMutedOrIgored();

if (history == null && !test.isMutedOrIgored()) {

if (test.getId() != null &&
newTestsStorage.isNewTest(tcIgnited.serverCode(),
test.getId(), failRateNormalizedBranch, ctx.branchName()))
return true;
else
return false;
}
else
return false;
})
.stream()
.map(occurrence -> {

0 comments on commit 728d98c

Please sign in to comment.