Skip to content
Permalink
Browse files
Cache and log cleaner (#162)
Signed-off-by: Ivan Rakov <ivan.glukos@gmail.com>
  • Loading branch information
sergeyuttsel authored and glukos committed Jun 3, 2020
1 parent 6d6da9d commit d46d69427c15d822244bb579570183e844c9c317
Showing 18 changed files with 295 additions and 0 deletions.
@@ -32,7 +32,9 @@
import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
import org.apache.ignite.tcbot.common.conf.ITcServerConfig;
import org.apache.ignite.tcbot.common.conf.TcBotWorkDir;
import org.apache.ignite.tcbot.engine.conf.CleanerConfig;
import org.apache.ignite.tcbot.engine.conf.GitHubConfig;
import org.apache.ignite.tcbot.engine.conf.ICleanerConfig;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.conf.ITrackedBranchesConfig;
import org.apache.ignite.tcbot.engine.conf.JiraServerConfig;
@@ -142,4 +144,10 @@ protected Properties loadOldAuthProps(String srvCode) {
return HelperConfig.loadAuthProperties(workDir, cfgName);
}

/** {@inheritDoc} */
@Override public ICleanerConfig getCleanerConfig() {
CleanerConfig cfg = getConfig().getCleanerConfig();
return cfg != null ? cfg : CleanerConfig.getDefaultCleanerConfig();
}

}
@@ -32,6 +32,7 @@
import javax.ws.rs.core.Form;
import javax.ws.rs.core.MediaType;
import org.apache.ignite.ci.tcbot.ITcBotBgAuth;
import org.apache.ignite.tcbot.engine.cleaner.Cleaner;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.ci.tcbot.issue.IssueDetector;
import org.apache.ignite.tcbot.engine.user.IUserStorage;
@@ -105,6 +106,9 @@ public SimpleResult setAuthorizedState() {

CtxListener.getInjector(ctx).getInstance(TcBotTriggerAndSignOffService.class).startObserver();

Cleaner cleaner = injector.getInstance(Cleaner.class);
cleaner.startBackgroundClean();

return userMenu(prov,
injector.getInstance(IUserStorage.class),
issueDetector);
@@ -29,6 +29,8 @@
import org.apache.ignite.jiraignited.IJiraIgnitedProvider;
import org.apache.ignite.jiraservice.IJiraIntegration;
import org.apache.ignite.jiraservice.IJiraIntegrationProvider;
import org.apache.ignite.tcbot.engine.conf.CleanerConfig;
import org.apache.ignite.tcbot.engine.conf.ICleanerConfig;
import org.apache.ignite.tcbot.engine.conf.TcBotJsonConfig;
import org.apache.ignite.githubservice.IGitHubConnection;
import org.apache.ignite.githubservice.IGitHubConnectionProvider;
@@ -123,6 +125,10 @@ public MockBasedTcBotModule() {
@Override public NotificationsConfig notifications() {
return new NotificationsConfig();
}

@Override public ICleanerConfig getCleanerConfig() {
return CleanerConfig.getDefaultCleanerConfig();
}
};
bind(ITcBotConfig.class).toInstance(cfg);
bind(IDataSourcesConfigSupplier.class).toInstance(cfg);
@@ -0,0 +1,115 @@
package org.apache.ignite.tcbot.engine.cleaner;

import java.io.File;
import java.time.ZonedDateTime;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildConditionDao;
import org.apache.ignite.tcbot.common.conf.TcBotWorkDir;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.defect.DefectsStorage;
import org.apache.ignite.tcbot.engine.issue.IIssuesStorage;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.buildlog.BuildLogCheckResultDao;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.tcignited.history.BuildStartTimeStorage;
import org.apache.ignite.tcignited.history.SuiteInvocationHistoryDao;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class Cleaner {
@Inject IIssuesStorage issuesStorage;
@Inject FatBuildDao fatBuildDao;
@Inject SuiteInvocationHistoryDao suiteInvocationHistoryDao;
@Inject BuildLogCheckResultDao buildLogCheckResultDao;
@Inject BuildRefDao buildRefDao;
@Inject BuildStartTimeStorage buildStartTimeStorage;
@Inject BuildConditionDao buildConditionDao;
@Inject DefectsStorage defectsStorage;
@Inject ITcBotConfig cfg;

/** Logger. */
private static final Logger logger = LoggerFactory.getLogger(Cleaner.class);

private ScheduledExecutorService executorService;

@AutoProfiling
@MonitoredTask(name = "Clean ignite cache data and logs")
public void clean() {
if (cfg.getCleanerConfig().enabled()) {
try {
long safeDays = cfg.getCleanerConfig().safeDays();
int numOfItemsToDel = cfg.getCleanerConfig().numOfItemsToDel();
long thresholdDate = ZonedDateTime.now().minusDays(safeDays).toInstant().toEpochMilli();

removeCacheEntries(thresholdDate, numOfItemsToDel);
removeLogFiles(thresholdDate, numOfItemsToDel);
}
catch (Exception e) {
e.printStackTrace();

logger.error("Periodic cache clean failed: " + e.getMessage(), e);
}
}
else
logger.info("Periodic cache clean disabled.");

}

private void removeCacheEntries(long thresholdDate, int numOfItemsToDel) {
List<Long> oldBuildsKeys = fatBuildDao.getOldBuilds(thresholdDate, numOfItemsToDel);

for (Long buildCacheKey : oldBuildsKeys) {
suiteInvocationHistoryDao.remove(buildCacheKey);
buildLogCheckResultDao.remove(buildCacheKey);
buildRefDao.remove(buildCacheKey);
buildStartTimeStorage.remove(buildCacheKey);
buildConditionDao.remove(buildCacheKey);
fatBuildDao.remove(buildCacheKey);
}

defectsStorage.removeOldDefects(thresholdDate, numOfItemsToDel);
issuesStorage.removeOldIssues(thresholdDate, numOfItemsToDel);
}

private void removeLogFiles(long thresholdDate, int numOfItemsToDel) {
final File workDir = TcBotWorkDir.resolveWorkDir();

for (String srvId : cfg.getServerIds()) {
File srvIdLogDir = new File(workDir, cfg.getTeamcityConfig(srvId).logsDirectory());
removeFiles(srvIdLogDir, thresholdDate, numOfItemsToDel);
}

File tcBotLogDir = new File(workDir, "tcbot_logs");
removeFiles(tcBotLogDir, thresholdDate, numOfItemsToDel);
}

private void removeFiles(File dir, long thresholdDate, int numOfItemsToDel) {
File[] logFiles = dir.listFiles();
if (logFiles != null)
for (File file : logFiles) {
if (file.lastModified() < thresholdDate && numOfItemsToDel-- > 0)
file.delete();
}
}

public void startBackgroundClean() {
suiteInvocationHistoryDao.init();
buildLogCheckResultDao.init();
buildRefDao.init();
buildStartTimeStorage.init();
buildConditionDao.init();
fatBuildDao.init();

executorService = Executors.newSingleThreadScheduledExecutor();

executorService.scheduleAtFixedRate(this::clean, 30, 30, TimeUnit.MINUTES);
// executorService.scheduleAtFixedRate(this::clean, 0, 10, TimeUnit.SECONDS);

}
}
@@ -0,0 +1,41 @@
package org.apache.ignite.tcbot.engine.conf;

public class CleanerConfig implements ICleanerConfig{
/** */
public static final int DEFAULT_SAVE_DAYS = 30 * 6;

/** */
public static final int DEFAULT_NUMBER_OF_ITEMS_TO_DELETE = 100;

/** */
private Integer safeDays;

/** */
private Integer numOfItemsToDel;

/** */
private Boolean enabled;

public static CleanerConfig getDefaultCleanerConfig() {
CleanerConfig cfg = new CleanerConfig();
cfg.safeDays = DEFAULT_SAVE_DAYS;
cfg.numOfItemsToDel = DEFAULT_NUMBER_OF_ITEMS_TO_DELETE;
cfg.enabled = true;
return cfg;
}

/** */
public int safeDays() {
return safeDays == null || safeDays < 0 ? DEFAULT_SAVE_DAYS : safeDays;
}

/** */
public int numOfItemsToDel() {
return numOfItemsToDel == null || numOfItemsToDel < 0 ? DEFAULT_NUMBER_OF_ITEMS_TO_DELETE : numOfItemsToDel;
}

/** */
public boolean enabled() {
return enabled == null ? true : enabled;
}
}
@@ -0,0 +1,12 @@
package org.apache.ignite.tcbot.engine.conf;

public interface ICleanerConfig {
/** */
int safeDays();

/** */
int numOfItemsToDel();

/** */
boolean enabled();
}
@@ -67,4 +67,6 @@ public default Collection<String> getServerIds() {
* @return notification settings config.
*/
public NotificationsConfig notifications();

public ICleanerConfig getCleanerConfig();
}
@@ -53,6 +53,8 @@ public class TcBotJsonConfig implements ITrackedBranchesConfig {
/** JIRA config to be used . */
private List<GitHubConfig> gitHubConfigs = new ArrayList<>();

private CleanerConfig cleanerConfig;

/** Notifications settings & tokens. */
private NotificationsConfig notifications = new NotificationsConfig();

@@ -115,6 +117,10 @@ public Optional<GitHubConfig> getGitHubConfig(String code) {
return gitHubConfigs.stream().filter(s -> Objects.equals(code, s.getCode())).findAny();
}

public CleanerConfig getCleanerConfig() {
return cleanerConfig;
}

@Nullable
public NotificationsConfig notifications() {
return notifications;
@@ -177,6 +177,10 @@ public void removeOldVerBlameCandidates() {
blameCandidates.removeIf(IVersionedEntity::isOutdatedEntityVersion);
}

public long resolvedTs() {
return resolvedTs;
}

/** {@inheritDoc} */
@Override public boolean equals(Object o) {
if (this == o)
@@ -30,6 +30,7 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteAtomicSequence;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.cache.QueryEntity;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
@@ -173,4 +174,23 @@ public void save(DefectCompacted defect) {
Preconditions.checkState(defect.id() != 0);
cache().put(defect.id(), defect);
}

public void removeOldDefects(long thresholdDate, int numOfItemsToDel) {
IgniteCache<Integer, BinaryObject> cacheWithBinary = cache().withKeepBinary();

ScanQuery<Integer, BinaryObject> scan =
new ScanQuery<>((key, defect) -> {
long resolvedTs = defect.<Long>field("resolvedTs");
return resolvedTs > 0 && resolvedTs < thresholdDate;
});

for (Cache.Entry<Integer, BinaryObject> entry : cacheWithBinary.query(scan)) {
if (numOfItemsToDel > 0) {
numOfItemsToDel--;
cacheWithBinary.remove(entry.getKey());
}
else
break;
}
}
}
@@ -45,4 +45,6 @@ public interface IIssuesStorage {
public boolean getIsNewAndSetNotified(IssueKey key, String addr, @Nullable Exception e);

public void saveIssueSubscribersStat(IssueKey key, int cntSrvAllowed, int cntSubscribed, int cntTagsFilterPassed);

public void removeOldIssues(long thresholdDate, int numOfItemsToDel);
}
@@ -26,6 +26,8 @@
import javax.inject.Provider;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.ci.issue.Issue;
import org.apache.ignite.ci.issue.IssueKey;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
@@ -125,4 +127,23 @@ public void saveIssueSubscribersStat(IssueKey issueKey, int cntSrvAllowed, int c
@Override public Stream<Issue> allIssues() {
return StreamSupport.stream(cache().spliterator(), false).map(Cache.Entry::getValue);
}

public void removeOldIssues(long thresholdDate, int numOfItemsToDel) {
IgniteCache<BinaryObject, BinaryObject> cacheWithBinary = cache().withKeepBinary();

ScanQuery<BinaryObject, BinaryObject> scan =
new ScanQuery<>((issueKey, issue) -> {
Long detectedTs = issue.<Long>field("detectedTs");
return detectedTs != null && detectedTs < thresholdDate && detectedTs > 0;
});

for (Cache.Entry<BinaryObject, BinaryObject> entry : cacheWithBinary.query(scan)) {
if (numOfItemsToDel > 0) {
numOfItemsToDel--;
cacheWithBinary.remove(entry.getKey());
}
else
break;
}
}
}
@@ -75,4 +75,8 @@ public boolean setBuildCondition(long srvIdMaskHigh, BuildCondition cond) {

return false;
}

public void remove(long key) {
buildsCache.remove(key);
}
}

0 comments on commit d46d694

Please sign in to comment.