Skip to content
Permalink
Browse files
Increased the number of files to save logs in 2 weeks #168
Signed-off-by: Ivan Rakov <ivan.glukos@gmail.com>
  • Loading branch information
sergeyuttsel authored and glukos committed Jul 9, 2020
1 parent 272ad28 commit b745f638c715d232b1c40c10647e285ec2f912e8
Showing 6 changed files with 103 additions and 33 deletions.
@@ -56,7 +56,7 @@ public static void configLogger(File workDir, String subdir) {
logFilePolicy.setContext(logCtx);
logFilePolicy.setParent(rollingFa);
logFilePolicy.setFileNamePattern(new File(logs, "logfile-%d{yyyy-MM-dd_HH}.log").getAbsolutePath());
logFilePolicy.setMaxHistory(7);
logFilePolicy.setMaxHistory(24*7*2);
logFilePolicy.start();

final String activeFileName = logFilePolicy.getActiveFileName();
@@ -38,6 +38,7 @@
import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.tcbot.common.exeption.ServicesStartingException;
import org.apache.ignite.tcbot.engine.TcBotEngineModule;
import org.apache.ignite.tcbot.engine.cleaner.Cleaner;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.pool.TcUpdatePool;
import org.apache.ignite.tcbot.notify.TcBotNotificationsModule;
@@ -78,6 +79,7 @@ public class TcBotWebAppModule extends AbstractModule {
bind(ObserverTask.class).in(new SingletonScope());
bind(BuildObserver.class).in(new SingletonScope());
bind(VisasHistoryStorage.class).in(new SingletonScope());
bind(Cleaner.class).in(new SingletonScope());

install(new TcBotPersistenceModule());
install(new TeamcityIgnitedModule());
@@ -18,12 +18,16 @@

import java.io.File;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import javax.inject.Inject;
import org.apache.ignite.ci.teamcity.ignited.buildcondition.BuildConditionDao;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.tcbot.common.conf.TcBotWorkDir;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
@@ -39,6 +43,8 @@
import org.slf4j.LoggerFactory;

public class Cleaner {
private final AtomicBoolean init = new AtomicBoolean();

@Inject IIssuesStorage issuesStorage;
@Inject FatBuildDao fatBuildDao;
@Inject SuiteInvocationHistoryDao suiteInvocationHistoryDao;
@@ -55,41 +61,62 @@ public class Cleaner {
private ScheduledExecutorService executorService;

@AutoProfiling
@MonitoredTask(name = "Clean ignite cache data and logs")
@MonitoredTask(name = "Clean old cache data and log files")
public void clean() {
if (cfg.getCleanerConfig().enabled()) {
try {
try {
if (cfg.getCleanerConfig().enabled()) {
long safeDays = cfg.getCleanerConfig().safeDays();

int numOfItemsToDel = cfg.getCleanerConfig().numOfItemsToDel();
long thresholdDate = ZonedDateTime.now().minusDays(safeDays).toInstant().toEpochMilli();

removeCacheEntries(thresholdDate, numOfItemsToDel);
removeLogFiles(thresholdDate, numOfItemsToDel);
}
catch (Exception e) {
e.printStackTrace();
ZonedDateTime thresholdDate = ZonedDateTime.now().minusDays(safeDays);

logger.info("Some data (numOfItemsToDel=" + numOfItemsToDel + ") older than " + thresholdDate + " will be removed.");

logger.error("Periodic cache clean failed: " + e.getMessage(), e);
long thresholdEpochMilli = thresholdDate.toInstant().toEpochMilli();

removeCacheEntries(thresholdEpochMilli, numOfItemsToDel);

removeLogFiles(thresholdEpochMilli, numOfItemsToDel);
}
else
logger.info("Periodic cache clean disabled.");
}
catch (Throwable e) {
logger.error("Periodic cache clean failed: " + e.getMessage(), e);

e.printStackTrace();
}
else
logger.info("Periodic cache clean disabled.");

}

private void removeCacheEntries(long thresholdDate, int numOfItemsToDel) {
List<Long> oldBuildsKeys = fatBuildDao.getOldBuilds(thresholdDate, numOfItemsToDel);

List<String> strOldBuildsKeys = oldBuildsKeys.stream().map(compositeId -> {
IgniteBiTuple<Integer, Integer> idTuple = FatBuildDao.cacheKeyToSrvIdAndBuildId(compositeId);
return "TeamCity id: " + idTuple.get1() + " build id: " + idTuple.get2();
})
.collect(Collectors.toList());

logger.info("Builds will be removed (" + strOldBuildsKeys.size() + "): " + strOldBuildsKeys);

for (Long buildCacheKey : oldBuildsKeys) {
suiteInvocationHistoryDao.remove(buildCacheKey);

buildLogCheckResultDao.remove(buildCacheKey);

buildRefDao.remove(buildCacheKey);

buildStartTimeStorage.remove(buildCacheKey);

buildConditionDao.remove(buildCacheKey);

fatBuildDao.remove(buildCacheKey);
}

defectsStorage.removeOldDefects(thresholdDate, numOfItemsToDel);

issuesStorage.removeOldIssues(thresholdDate, numOfItemsToDel);
}

@@ -98,34 +125,47 @@ private void removeLogFiles(long thresholdDate, int numOfItemsToDel) {

for (String srvId : cfg.getServerIds()) {
File srvIdLogDir = new File(workDir, cfg.getTeamcityConfig(srvId).logsDirectory());

removeFiles(srvIdLogDir, thresholdDate, numOfItemsToDel);
}

File tcBotLogDir = new File(workDir, "tcbot_logs");

removeFiles(tcBotLogDir, thresholdDate, numOfItemsToDel);
}

private void removeFiles(File dir, long thresholdDate, int numOfItemsToDel) {
File[] logFiles = dir.listFiles();

List<File> filesToRmv = new ArrayList<>(numOfItemsToDel);

if (logFiles != null)
for (File file : logFiles) {
for (File file : logFiles)
if (file.lastModified() < thresholdDate && numOfItemsToDel-- > 0)
file.delete();
}
filesToRmv.add(file);

logger.info("In the directory " + dir + " files will be removed (" +
filesToRmv.size() + "): " + filesToRmv.stream().map(File::getName).collect(Collectors.toList())
);

for (File file : filesToRmv) {
file.delete();
}

}

public void startBackgroundClean() {
suiteInvocationHistoryDao.init();
buildLogCheckResultDao.init();
buildRefDao.init();
buildStartTimeStorage.init();
buildConditionDao.init();
fatBuildDao.init();

executorService = Executors.newSingleThreadScheduledExecutor();
if (init.compareAndSet(false, true)) {
suiteInvocationHistoryDao.init();
buildLogCheckResultDao.init();
buildRefDao.init();
buildStartTimeStorage.init();
buildConditionDao.init();
fatBuildDao.init();

executorService.scheduleAtFixedRate(this::clean, 30, 30, TimeUnit.MINUTES);
// executorService.scheduleAtFixedRate(this::clean, 0, 10, TimeUnit.SECONDS);
executorService = Executors.newSingleThreadScheduledExecutor();

executorService.scheduleAtFixedRate(this::clean, 5, 30, TimeUnit.MINUTES);
}
}
}
@@ -180,8 +180,13 @@ public void removeOldDefects(long thresholdDate, int numOfItemsToDel) {

ScanQuery<Integer, BinaryObject> scan =
new ScanQuery<>((key, defect) -> {
long resolvedTs = defect.<Long>field("resolvedTs");
return resolvedTs > 0 && resolvedTs < thresholdDate;
Long resolvedTs = 0L;

if (defect.hasField("resolvedTs"))
resolvedTs = defect.<Long>field("resolvedTs");

return (resolvedTs > 0 && resolvedTs < thresholdDate) ||
!defect.hasField("resolvedTs");
});

for (Cache.Entry<Integer, BinaryObject> entry : cacheWithBinary.query(scan)) {
@@ -133,8 +133,13 @@ public void removeOldIssues(long thresholdDate, int numOfItemsToDel) {

ScanQuery<BinaryObject, BinaryObject> scan =
new ScanQuery<>((issueKey, issue) -> {
Long detectedTs = issue.<Long>field("detectedTs");
return detectedTs != null && detectedTs < thresholdDate && detectedTs > 0;
Long detectedTs = null;

if (issue.hasField("detectedTs"))
detectedTs = issue.<Long>field("detectedTs");

return (detectedTs != null && detectedTs < thresholdDate && detectedTs > 0) ||
!issue.hasField("detectedTs");
});

for (Cache.Entry<BinaryObject, BinaryObject> entry : cacheWithBinary.query(scan)) {
@@ -19,6 +19,7 @@

import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
@@ -47,6 +48,7 @@
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.lang.IgniteBiPredicate;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.CacheConfigs;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
@@ -176,6 +178,17 @@ public static long buildIdToCacheKey(int srvIdMaskHigh, int buildId) {
return (long)buildId | (long)srvIdMaskHigh << 32;
}

/**
* @param cacheKey cache key.
*/
public static IgniteBiTuple<Integer, Integer> cacheKeyToSrvIdAndBuildId(long cacheKey) {
IgniteBiTuple<Integer, Integer> srvIdAndBuildId = new IgniteBiTuple<>();

srvIdAndBuildId.set((int)(cacheKey >> 32), (int)cacheKey);

return srvIdAndBuildId;
}

/**
* @param srvIdMaskHigh Server id mask high.
* @param buildId Build id.
@@ -410,12 +423,18 @@ public List<Long> getOldBuilds(long thresholdDate, int numOfItemsToDel) {
IgniteCache<Long, BinaryObject> cacheWithBinary = buildsCache.withKeepBinary();

ScanQuery<Long, BinaryObject> scan = new ScanQuery<>((key, fatBuild) -> {
long startDate = fatBuild.<Long>field("startDate");
return startDate > 0 && startDate < thresholdDate;
Long startDate = 0L;

if (fatBuild.hasField("startDate"))
startDate = fatBuild.<Long>field("startDate");

return (startDate > 0 && startDate < thresholdDate) ||
!fatBuild.hasField("startDate");
}
);

List<Long> oldBuildsKeys = new ArrayList<>(numOfItemsToDel);

for (Cache.Entry<Long, BinaryObject> entry : cacheWithBinary.query(scan)) {
if (numOfItemsToDel > 0) {
numOfItemsToDel--;
@@ -445,5 +464,4 @@ private static class IsMissingBuildProcessor implements CacheEntryProcessor<Long
return entry.getValue() == null ? BuildRefDao.cacheKeyToBuildId(key) : null;
}
}

}

0 comments on commit b745f63

Please sign in to comment.