Skip to content
Permalink
Browse files
TC Bot engine module introduced: tracked branch processor moved to en…
…gine
  • Loading branch information
dspavlov committed Jun 9, 2019
1 parent 6b72596 commit c8021f443a654b3269175c036c316decdea90235
Showing 39 changed files with 369 additions and 338 deletions.
@@ -39,9 +39,8 @@
import org.apache.ignite.tcbot.persistence.TcBotPersistenceModule;
import org.apache.ignite.tcignited.TeamcityIgnitedModule;
import org.apache.ignite.tcbot.common.exeption.ExceptionUtil;
import org.apache.ignite.ci.web.TcUpdatePool;
import org.apache.ignite.tcbot.engine.pool.TcUpdatePool;
import org.apache.ignite.ci.web.model.hist.VisasHistoryStorage;
import org.apache.ignite.ci.web.rest.exception.ServiceStartingExceptionMapper;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;

@@ -37,7 +37,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.ignite.ci.util.UrlUtil.escape;
import static org.apache.ignite.tcbot.common.util.UrlUtil.escape;

/**
* Sync serving requests for all JIRA servers.
@@ -19,7 +19,7 @@
import com.google.inject.AbstractModule;
import com.google.inject.internal.SingletonScope;
import org.apache.ignite.ci.issue.IssuesStorage;
import org.apache.ignite.ci.tcbot.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.ci.tcbot.conf.LocalFilesBasedConfig;
import org.apache.ignite.ci.tcbot.issue.IIssuesStorage;
@@ -21,7 +21,7 @@
import java.util.List;
import javax.inject.Inject;
import org.apache.ignite.tcbot.engine.chain.MultBuildRunCtx;
import org.apache.ignite.ci.tcbot.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
import org.apache.ignite.tcignited.ITeamcityIgnited;
import org.apache.ignite.tcignited.ITeamcityIgnitedProvider;
@@ -22,20 +22,20 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import javax.inject.Inject;
import org.apache.ignite.tcbot.engine.chain.FullChainRunCtx;
import org.apache.ignite.tcbot.engine.chain.TestCompactedMult;
import org.apache.ignite.tcbot.engine.chain.LatestRebuildMode;
import org.apache.ignite.tcbot.engine.chain.ProcessLogsMode;

import org.apache.ignite.ci.github.PullRequest;
import org.apache.ignite.ci.github.pure.IGitHubConnection;
import org.apache.ignite.tcbot.engine.chain.*;
import org.apache.ignite.ci.github.ignited.IGitHubConnIgnited;
import org.apache.ignite.ci.github.ignited.IGitHubConnIgnitedProvider;
import org.apache.ignite.ci.jira.ignited.IJiraIgnited;
import org.apache.ignite.ci.jira.ignited.IJiraIgnitedProvider;
import org.apache.ignite.ci.tcbot.visa.BranchTicketMatcher;
import org.apache.ignite.ci.user.ITcBotUserCreds;
import org.apache.ignite.ci.web.model.current.ChainAtServerCurrentStatus;
import org.apache.ignite.ci.web.model.current.SuiteCurrentStatus;
import org.apache.ignite.ci.web.model.current.TestFailure;
import org.apache.ignite.ci.web.model.current.TestFailuresSummary;
import org.apache.ignite.tcbot.engine.ui.DsChainUi;
import org.apache.ignite.tcbot.engine.ui.DsSuiteUi;
import org.apache.ignite.tcbot.engine.ui.DsTestFailureUi;
import org.apache.ignite.tcbot.engine.ui.DsSummaryUi;
import org.apache.ignite.ci.web.rest.parms.FullQueryParams;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.persistence.IStringCompactor;
@@ -80,7 +80,7 @@ public class PrChainsProcessor {
* @return Test failures summary.
*/
@AutoProfiling
public TestFailuresSummary getTestFailuresSummary(
public DsSummaryUi getTestFailuresSummary(
ITcBotUserCreds creds,
String srvCode,
String suiteId,
@@ -90,7 +90,7 @@ public TestFailuresSummary getTestFailuresSummary(
@Nullable String baseBranchForTc,
@Nullable Boolean checkAllLogs,
SyncMode mode) {
final TestFailuresSummary res = new TestFailuresSummary();
final DsSummaryUi res = new DsSummaryUi();
final AtomicInteger runningUpdates = new AtomicInteger();

//using here non persistent TC allows to skip update statistic
@@ -100,7 +100,7 @@ public TestFailuresSummary getTestFailuresSummary(

IJiraIgnited jiraIntegration = jiraIgnProv.server(srvCode);

res.setJavaFlags(gitHubConnIgnited, jiraIntegration);
res.setJavaFlags(gitHubConnIgnited.config(), jiraIntegration.config());

LatestRebuildMode rebuild;
if (FullQueryParams.HISTORY.equals(act))
@@ -137,7 +137,7 @@ else if (FullQueryParams.CHAIN.equals(act))
baseBranch,
mode);

ChainAtServerCurrentStatus chainStatus = new ChainAtServerCurrentStatus(srvCode, tcIgnited.serverCode(), branchForTc);
DsChainUi chainStatus = new DsChainUi(srvCode, tcIgnited.serverCode(), branchForTc);

chainStatus.baseBranchForTc = baseBranch;

@@ -152,7 +152,7 @@ else if (FullQueryParams.CHAIN.equals(act))
//fail rate reference is always default (master)
chainStatus.initFromContext(tcIgnited, ctx, baseBranch, compactor, false); // don't need for PR

chainStatus.initJiraAndGitInfo(ticketMatcher, jiraIntegration, gitHubConnIgnited);
initJiraAndGitInfo(chainStatus, jiraIntegration, gitHubConnIgnited);
}

res.addChainOnServer(chainStatus);
@@ -162,23 +162,63 @@ else if (FullQueryParams.CHAIN.equals(act))
return res;
}

/**
* Set up ticket and PR related information.
* @param chainStatus Ticket matcher.
* @param jiraIntegration Jira integration.
* @param gitHubConnIgnited Git hub connection ignited.
*/
public void initJiraAndGitInfo(DsChainUi chainStatus,
IJiraIgnited jiraIntegration,
IGitHubConnIgnited gitHubConnIgnited) {

String ticketFullName = null;
String branchName = chainStatus.branchName;
try {
ticketFullName = ticketMatcher
.resolveTicketFromBranch(jiraIntegration.config().getCode(),
null,
branchName);
}
catch (BranchTicketMatcher.TicketNotFoundException ignore) {
}

Integer prNum = IGitHubConnection.convertBranchToPrId(branchName);

String prUrl = null;
String ticketUrl = null;

if (prNum != null) {
PullRequest pullReq = gitHubConnIgnited.getPullRequest(prNum);

if (pullReq != null && pullReq.getTitle() != null)
prUrl = pullReq.htmlUrl();
}

if (!Strings.isNullOrEmpty(ticketFullName) && jiraIntegration.config().getUrl() != null)
ticketUrl = jiraIntegration.generateTicketUrl(ticketFullName);

chainStatus.setPrInfo(prNum, prUrl);
chainStatus.setJiraTicketInfo(ticketFullName, ticketUrl);
}

/**
* @param buildTypeId Build type ID, for which visa was ordered.
* @param branchForTc Branch for TeamCity.
* @param srvId Server id.
* @param prov Credentials.
* @return List of suites with possible blockers.
*/
@Nullable public List<SuiteCurrentStatus> getBlockersSuitesStatuses(String buildTypeId,
String branchForTc,
String srvId,
ITcBotUserCreds prov) {
@Nullable public List<DsSuiteUi> getBlockersSuitesStatuses(String buildTypeId,
String branchForTc,
String srvId,
ITcBotUserCreds prov) {
return getBlockersSuitesStatuses(buildTypeId, branchForTc, srvId, prov, SyncMode.RELOAD_QUEUED);
}

@Nullable
public List<SuiteCurrentStatus> getBlockersSuitesStatuses(String buildTypeId, String branchForTc, String srvId,
ITcBotUserCreds prov, SyncMode syncMode) {
public List<DsSuiteUi> getBlockersSuitesStatuses(String buildTypeId, String branchForTc, String srvId,
ITcBotUserCreds prov, SyncMode syncMode) {
//using here non persistent TC allows to skip update statistic
ITeamcityIgnited tcIgnited = tcIgnitedProvider.server(srvId, prov);

@@ -208,8 +248,8 @@ public List<SuiteCurrentStatus> getBlockersSuitesStatuses(String buildTypeId, St
* @param baseBranch
*/
//todo may avoid creation of UI model for simple comment.
private List<SuiteCurrentStatus> findBlockerFailures(FullChainRunCtx fullChainRunCtx, ITeamcityIgnited tcIgnited,
String baseBranch) {
private List<DsSuiteUi> findBlockerFailures(FullChainRunCtx fullChainRunCtx, ITeamcityIgnited tcIgnited,
String baseBranch) {
return fullChainRunCtx
.failedChildSuites()
.map((ctx) -> {
@@ -218,13 +258,13 @@ private List<SuiteCurrentStatus> findBlockerFailures(FullChainRunCtx fullChainRu

String suiteComment = ctx.getPossibleBlockerComment(compactor, statInBaseBranch, tcIgnited.config());

List<TestFailure> failures = ctx.getFailedTests().stream().map(occurrence -> {
List<DsTestFailureUi> failures = ctx.getFailedTests().stream().map(occurrence -> {
IRunHistory stat = tcIgnited.getTestRunHist(occurrence.getName(), normalizedBaseBranch);

String testBlockerComment = TestCompactedMult.getPossibleBlockerComment(stat);

if (!Strings.isNullOrEmpty(testBlockerComment)) {
final TestFailure failure = new TestFailure();
final DsTestFailureUi failure = new DsTestFailureUi();

failure.initFromOccurrence(occurrence, tcIgnited, ctx.projectId(), ctx.branchName(), baseBranch);

@@ -237,7 +277,7 @@ private List<SuiteCurrentStatus> findBlockerFailures(FullChainRunCtx fullChainRu
// test failure based blockers and/or blocker found by suite results
if (!failures.isEmpty() || !Strings.isNullOrEmpty(suiteComment)) {

SuiteCurrentStatus suiteUi = new SuiteCurrentStatus();
DsSuiteUi suiteUi = new DsSuiteUi();
suiteUi.testFailures = failures;

suiteUi.initFromContext(tcIgnited, ctx, baseBranch, compactor, false, false);
@@ -23,17 +23,19 @@
import org.apache.ignite.ci.jobs.CheckQueueJob;
import org.apache.ignite.ci.mail.EmailSender;
import org.apache.ignite.ci.mail.SlackSender;
import org.apache.ignite.ci.tcbot.chain.TrackedBranchChainsProcessor;
import org.apache.ignite.tcbot.engine.issue.EventTemplate;
import org.apache.ignite.tcbot.engine.issue.EventTemplates;
import org.apache.ignite.tcbot.engine.tracked.TrackedBranchChainsProcessor;
import org.apache.ignite.ci.tcbot.user.IUserStorage;
import org.apache.ignite.ci.teamcity.ignited.change.ChangeCompacted;
import org.apache.ignite.ci.teamcity.ignited.fatbuild.FatBuildCompacted;
import org.apache.ignite.ci.teamcity.ignited.runhist.InvocationData;
import org.apache.ignite.ci.user.ITcBotUserCreds;
import org.apache.ignite.ci.user.TcHelperUser;
import org.apache.ignite.ci.web.model.current.ChainAtServerCurrentStatus;
import org.apache.ignite.ci.web.model.current.SuiteCurrentStatus;
import org.apache.ignite.ci.web.model.current.TestFailure;
import org.apache.ignite.ci.web.model.current.TestFailuresSummary;
import org.apache.ignite.tcbot.engine.ui.DsChainUi;
import org.apache.ignite.tcbot.engine.ui.DsSuiteUi;
import org.apache.ignite.tcbot.engine.ui.DsTestFailureUi;
import org.apache.ignite.tcbot.engine.ui.DsSummaryUi;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.MonitoredTask;
import org.apache.ignite.tcbot.engine.conf.INotificationChannel;
@@ -96,8 +98,8 @@ public class IssueDetector {
/** Send notification guard. */
private final AtomicBoolean sndNotificationGuard = new AtomicBoolean();

private String registerIssuesAndNotifyLater(TestFailuresSummary res,
ITcBotUserCreds creds) {
private String registerIssuesAndNotifyLater(DsSummaryUi res,
ITcBotUserCreds creds) {

if (creds == null)
return null;
@@ -243,23 +245,23 @@ protected String sendNewNotificationsEx() throws IOException {
@SuppressWarnings({"WeakerAccess", "UnusedReturnValue"})
@AutoProfiling
@MonitoredTask(name = "Register new issues")
protected String registerNewIssues(TestFailuresSummary res, ITcBotUserCreds creds) {
protected String registerNewIssues(DsSummaryUi res, ITcBotUserCreds creds) {
int newIssues = 0;

for (ChainAtServerCurrentStatus next : res.servers) {
for (DsChainUi next : res.servers) {
String srvCode = next.serverCode;

if (!tcProv.hasAccess(srvCode, creds))
continue;

ITeamcityIgnited tcIgnited = tcProv.server(srvCode, creds);

for (SuiteCurrentStatus suiteCurrentStatus : next.suites) {
for (DsSuiteUi suiteCurrentStatus : next.suites) {
String normalizeBranch = normalizeBranch(suiteCurrentStatus.branchName());

final String trackedBranch = res.getTrackedBranch();

for (TestFailure testFailure : suiteCurrentStatus.testFailures) {
for (DsTestFailureUi testFailure : suiteCurrentStatus.testFailures) {
if (registerTestFailIssues(tcIgnited, srvCode, normalizeBranch, testFailure, trackedBranch,
suiteCurrentStatus.tags))
newIssues++;
@@ -285,7 +287,7 @@ protected String registerNewIssues(TestFailuresSummary res, ITcBotUserCreds cred
private boolean registerSuiteFailIssues(ITeamcityIgnited tcIgnited,
String srvCode,
String normalizeBranch,
SuiteCurrentStatus suiteFailure,
DsSuiteUi suiteFailure,
String trackedBranch) {

String suiteId = suiteFailure.suiteId;
@@ -330,8 +332,8 @@ private boolean registerSuiteFailIssues(ITeamcityIgnited tcIgnited,
}

@NotNull
private Issue createIssueForSuite(ITeamcityIgnited tcIgnited, SuiteCurrentStatus suiteFailure, String trackedBranch,
IssueKey issueKey, IssueType issType) {
private Issue createIssueForSuite(ITeamcityIgnited tcIgnited, DsSuiteUi suiteFailure, String trackedBranch,
IssueKey issueKey, IssueType issType) {
Issue issue = new Issue(issueKey, issType);
issue.trackedBranchName = trackedBranch;
issue.displayName = suiteFailure.name;
@@ -360,7 +362,7 @@ private void locateChanges(ITeamcityIgnited teamcity, int buildId, Issue issue)
private boolean registerTestFailIssues(ITeamcityIgnited tcIgnited,
String srvCode,
String normalizeBranch,
TestFailure testFailure,
DsTestFailureUi testFailure,
String trackedBranch,
@Nonnull Set<String> suiteTags) {
String name = testFailure.name;
@@ -493,7 +495,7 @@ protected String checkFailuresEx(String brachName) {
SyncMode.RELOAD_QUEUED,
false);

TestFailuresSummary failures =
DsSummaryUi failures =
tbProc.getTrackedBranchTestFailures(brachName,
false,
1,
@@ -41,7 +41,7 @@
import javax.inject.Provider;
import org.apache.ignite.tcbot.common.interceptor.AutoProfiling;
import org.apache.ignite.tcbot.common.interceptor.GuavaCached;
import org.apache.ignite.ci.tcbot.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.chain.BuildChainProcessor;
import org.apache.ignite.tcbot.engine.conf.ITcBotConfig;
import org.apache.ignite.tcservice.model.hist.BuildRef;
import org.apache.ignite.tcservice.model.result.tests.TestOccurrence;

0 comments on commit c8021f4

Please sign in to comment.