Skip to content
Permalink
Browse files
Issue detection bug fixed: build TS problem fixed
  • Loading branch information
dspavlov committed Jul 1, 2019
1 parent 9801cc6 commit 5573467d04eedbe9f79dd2b19e9675b76af78985
Show file tree
Hide file tree
Showing 5 changed files with 111 additions and 46 deletions.
@@ -17,6 +17,8 @@

package org.apache.ignite.ci.issue;

import com.google.common.base.MoreObjects;

public class ChangeUi {
public final String username;
public final String webUrl;
@@ -41,4 +43,12 @@ public String toSlackMarkup() {
public String toPlainText() {
return username + " " + webUrl;
}

/** {@inheritDoc} */
@Override public String toString() {
return MoreObjects.toStringHelper(this)
.add("username", username)
.add("webUrl", webUrl)
.toString();
}
}
@@ -57,7 +57,7 @@ public class Issue {
@Nullable
public String displayName;

/** Build start timestamp. */
/** Build start timestamp. Builds which is older that 10 days not notified. */
@Nullable public Long buildStartTs;

/** Detected timestamp. */
@@ -66,11 +66,13 @@ public class Issue {
/** Set of build tags detected. */
public Set<String> buildTags = new TreeSet<>();

public Issue(IssueKey issueKey, IssueType type) {
public Issue(IssueKey issueKey, IssueType type,
@Nullable Long buildStartTs) {
this.issueKey = issueKey;
this.detectedTs = System.currentTimeMillis();
this.type = type.code();
this.displayType = type.displayName();
this.buildStartTs = buildStartTs;
}

public void addChange(String username, String webUrl) {
@@ -19,9 +19,9 @@

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.time.Duration;
@@ -32,22 +32,25 @@
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.ci.issue.Issue;
import org.apache.ignite.ci.issue.IssueKey;
import org.apache.ignite.ci.issue.IssuesStorage;
import org.apache.ignite.ci.tcbot.user.UserAndSessionsStorage;
import org.apache.ignite.tcservice.model.hist.BuildRef;
import org.apache.ignite.tcservice.model.result.Build;
import org.apache.ignite.ci.teamcity.ignited.BuildRefCompacted;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcbot.persistence.IgniteStringCompactor;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.history.RunHistCompactedDao;
import org.apache.ignite.ci.user.TcHelperUser;
import org.apache.ignite.tcservice.util.XmlUtil;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.logger.slf4j.Slf4jLogger;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.tcbot.persistence.IgniteStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.build.FatBuildDao;
import org.apache.ignite.tcignited.buildref.BuildRefDao;
import org.apache.ignite.tcignited.history.RunHistCompactedDao;
import org.apache.ignite.tcservice.model.hist.BuildRef;
import org.apache.ignite.tcservice.model.result.Build;
import org.apache.ignite.tcservice.util.XmlUtil;
import org.jetbrains.annotations.NotNull;

import static org.apache.ignite.tcignited.history.RunHistCompactedDao.BUILD_START_TIME_CACHE_NAME;
@@ -64,10 +67,38 @@ public class RemoteClientTmpHelper {
* @param args Args.
*/
public static void main(String[] args) {
// mainDumpFatBuildStartTime(args);
// mainDumpIssues(args);
System.err.println("Please insert option of main");
}

public static void mainDumpIssues(String[] args) {
try (Ignite ignite = tcbotServerConnectedClient()) {
IgniteCache<IssueKey, Issue> bst = ignite.cache(IssuesStorage.BOT_DETECTED_ISSUES);
Iterator<Cache.Entry<IssueKey, Issue>> iter = bst.iterator();
try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(dumpsDir(),
"Issues.txt")))) {
while (iter.hasNext()) {
Cache.Entry<IssueKey, Issue> next = iter.next();

Issue val = next.getValue();
long ageDays = -1;
if (val != null && val.buildStartTs != null)
ageDays = Duration.ofMillis(System.currentTimeMillis() - val.buildStartTs).toDays();

writer.write(next.getKey() + " " + val + " " +
ageDays + "\n");
}

writer.flush();
}
catch (IOException e) {
throw new UncheckedIOException(e);
}

dumpDictionary(ignite);
}
}


public static void mainDumpFatBuildStartTime(String[] args) {
try (Ignite ignite = tcbotServerConnectedClient()) {
@@ -255,18 +286,12 @@ private static void setupDisco(IgniteConfiguration cfg) {
cfg.setDiscoverySpi(spi);
}

public static void mainExport(String[] args) throws IOException {
public static void mainExport(String[] args) {
final Ignite ignite = tcbotServerConnectedClient();

if (dumpDict) {
IgniteCache<String, Object> strings = ignite.cache(IgniteStringCompactor.STRINGS_CACHE);
try (FileWriter writer = new FileWriter("Dictionary.txt")) {
for (Cache.Entry<String, Object> next1 : strings) {
writer.write(next1.getValue().toString()
+ "\n");
}
}
}
if (dumpDict)
dumpDictionary(ignite);

if (false) {
IgniteCache<Long, FatBuildCompacted> cache1 = ignite.cache(FatBuildDao.TEAMCITY_FAT_BUILD_CACHE_NAME);

@@ -283,6 +308,21 @@ public static void mainExport(String[] args) throws IOException {
}
}

public static void dumpDictionary(Ignite ignite) {
IgniteCache<String, Object> strings = ignite.cache(IgniteStringCompactor.STRINGS_CACHE);
try {
try (FileWriter writer = new FileWriter(new File(dumpsDir(), "Dictionary.txt"))) {
for (Cache.Entry<String, Object> next1 : strings) {
writer.write(next1.getValue().toString()
+ "\n");
}
}
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
}

public static Ignite tcbotServerConnectedClient() {
final IgniteConfiguration cfg = new IgniteConfiguration();

@@ -19,28 +19,45 @@

import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import org.apache.ignite.ci.issue.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nonnull;
import javax.inject.Inject;
import javax.inject.Provider;
import org.apache.ignite.ci.issue.Issue;
import org.apache.ignite.ci.issue.IssueKey;
import org.apache.ignite.ci.issue.IssueType;
import org.apache.ignite.ci.jobs.CheckQueueJob;
import org.apache.ignite.ci.mail.EmailSender;
import org.apache.ignite.ci.mail.SlackSender;
import org.apache.ignite.tcbot.engine.issue.EventTemplate;
import org.apache.ignite.tcbot.engine.issue.EventTemplates;
import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor;
import org.apache.ignite.ci.tcbot.user.IUserStorage;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.apache.ignite.ci.user.ITcBotUserCreds;
import org.apache.ignite.ci.user.TcHelperUser;
import org.apache.ignite.tcbot.engine.ui.DsChainUi;
import org.apache.ignite.tcbot.engine.ui.DsSuiteUi;
import org.apache.ignite.tcbot.engine.ui.DsTestFailureUi;
import org.apache.ignite.tcbot.engine.ui.DsSummaryUi;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
import org.apache.ignite.tcbot.engine.conf.INotificationChannel;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcbot.engine.conf.NotificationsConfig;
import org.apache.ignite.tcbot.engine.issue.EventTemplate;
import org.apache.ignite.tcbot.engine.issue.EventTemplates;
import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor;
import org.apache.ignite.tcbot.engine.ui.DsChainUi;
import org.apache.ignite.tcbot.engine.ui.DsSuiteUi;
import org.apache.ignite.tcbot.engine.ui.DsSummaryUi;
import org.apache.ignite.tcbot.engine.ui.DsTestFailureUi;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
@@ -50,17 +67,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nonnull;
import javax.inject.Inject;
import javax.inject.Provider;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;

import static org.apache.ignite.tcignited.history.RunHistSync.normalizeBranch;

/**
@@ -150,9 +156,17 @@ protected String sendNewNotificationsEx() throws IOException {
long detected = issue.detectedTs == null ? 0 : issue.detectedTs;
long issueAgeMs = System.currentTimeMillis() - detected;

return issueAgeMs <= TimeUnit.HOURS.toMillis(2);
long bound = TimeUnit.HOURS.toMillis(2);
//temporary to issue missed notifications
if (issue.buildStartTs == null)
bound = TimeUnit.DAYS.toMillis(InvocationData.MAX_DAYS) / 2;

return issueAgeMs <= bound;
})
.filter(issue -> {
if (issue.buildStartTs == null)
return true; // exception due to bug in issue detection; field was not filled

long buildStartTs = issue.buildStartTs == null ? 0 : issue.buildStartTs;
long buildAgeMs = System.currentTimeMillis() - buildStartTs;
long maxBuildAgeToNotify = TimeUnit.DAYS.toMillis(InvocationData.MAX_DAYS) / 2;
@@ -337,11 +351,10 @@ private boolean registerSuiteFailIssues(ITeamcityIgnited tcIgnited,
@NotNull
private Issue createIssueForSuite(ITeamcityIgnited tcIgnited, DsSuiteUi suiteFailure, String trackedBranch,
IssueKey issueKey, IssueType issType) {
Issue issue = new Issue(issueKey, issType);
Issue issue = new Issue(issueKey, issType, tcIgnited.getBuildStartTs(issueKey.buildId));
issue.trackedBranchName = trackedBranch;
issue.displayName = suiteFailure.name;
issue.webUrl = suiteFailure.webToHist;
issue.buildStartTs = tcIgnited.getBuildStartTs(issueKey.buildId);

issue.buildTags.addAll(suiteFailure.tags);

@@ -419,7 +432,7 @@ private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited,
if (issuesStorage.containsIssueKey(issueKey))
return false; //duplicate

Issue issue = new Issue(issueKey, type);
Issue issue = new Issue(issueKey, type, tcIgnited.getBuildStartTs(issueKey.buildId));
issue.trackedBranchName = trackedBranch;
issue.displayName = testFailure.testName;
issue.webUrl = testFailure.webUrl;
@@ -28,7 +28,7 @@
public static final String GITHUB_REF = "https://github.com/apache/ignite-teamcity-bot";

/** TC Bot Version. */
public static final String VERSION = "20190626";
public static final String VERSION = "20190701";

/** Java version, where Web App is running. */
public String javaVer;

0 comments on commit 5573467

Please sign in to comment.