Skip to content
Permalink
Browse files
Add census support for the issue notifier
Reviewed-by: erikj
  • Loading branch information
rwestberg committed Apr 20, 2021
1 parent 53bfe81 commit b8f101674e61df08cc451b2fbe02e8e669764c77
Showing with 312 additions and 129 deletions.
  1. +9 −6 bots/notify/src/main/java/org/openjdk/skara/bots/notify/PullRequestListener.java
  2. +21 −20 bots/notify/src/main/java/org/openjdk/skara/bots/notify/PullRequestWorkItem.java
  3. +5 −4 bots/notify/src/main/java/org/openjdk/skara/bots/notify/RepositoryListener.java
  4. +13 −12 bots/notify/src/main/java/org/openjdk/skara/bots/notify/RepositoryWorkItem.java
  5. +7 −1 bots/notify/src/main/java/org/openjdk/skara/bots/notify/comment/CommitCommentNotifier.java
  6. +84 −0 bots/notify/src/main/java/org/openjdk/skara/bots/notify/issue/CensusInstance.java
  7. +34 −14 bots/notify/src/main/java/org/openjdk/skara/bots/notify/issue/IssueNotifier.java
  8. +21 −1 bots/notify/src/main/java/org/openjdk/skara/bots/notify/issue/IssueNotifierBuilder.java
  9. +8 −0 bots/notify/src/main/java/org/openjdk/skara/bots/notify/issue/IssueNotifierFactory.java
  10. +2 −2 bots/notify/src/main/java/org/openjdk/skara/bots/notify/json/JsonNotifier.java
  11. +6 −5 bots/notify/src/main/java/org/openjdk/skara/bots/notify/mailinglist/MailingListNotifier.java
  12. +8 −3 bots/notify/src/main/java/org/openjdk/skara/bots/notify/prbranch/PullRequestBranchNotifier.java
  13. +3 −2 bots/notify/src/main/java/org/openjdk/skara/bots/notify/slack/SlackNotifier.java
  14. +8 −8 bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java
  15. +83 −6 bots/notify/src/test/java/org/openjdk/skara/bots/notify/issue/IssueNotifierTests.java
  16. +0 −2 issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueProject.java
  17. +0 −28 issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java
  18. +0 −9 test/src/main/java/org/openjdk/skara/test/TestHost.java
  19. +0 −6 test/src/main/java/org/openjdk/skara/test/TestIssueProject.java
@@ -26,17 +26,20 @@
import org.openjdk.skara.vcs.Hash;
import org.openjdk.skara.vcs.openjdk.Issue;

import java.nio.file.Path;

public interface PullRequestListener {
default void onNewIssue(PullRequest pr, Issue issue) {
default void onNewIssue(PullRequest pr, Path scratchPath, Issue issue) {
}
default void onRemovedIssue(PullRequest pr, Issue issue) {
default void onRemovedIssue(PullRequest pr, Path scratchPath, Issue issue) {
}
default void onNewPullRequest(PullRequest pr) {
default void onNewPullRequest(PullRequest pr, Path scratchPath) {
}
default void onIntegratedPullRequest(PullRequest pr, Hash hash) {
default void onIntegratedPullRequest(PullRequest pr, Path scratchPath, Hash hash) {
}
default void onHeadChange(PullRequest pr, Hash oldHead) {
default void onHeadChange(PullRequest pr, Path scratchPath, Hash oldHead) {
}
default void onStateChange(PullRequest pr, org.openjdk.skara.issuetracker.Issue.State oldState) {
default void onStateChange(PullRequest pr, Path scratchPath, org.openjdk.skara.issuetracker.Issue.State oldState) {
}
String name();
}
@@ -169,33 +169,34 @@ public boolean concurrentWith(WorkItem other) {
return false;
}

private void notifyNewIssue(String issueId) {
listeners.forEach(c -> c.onNewIssue(pr, new Issue(issueId, "")));
private void notifyNewIssue(String issueId, Path scratchPath) {
listeners.forEach(c -> c.onNewIssue(pr, scratchPath.resolve(c.name()), new Issue(issueId, "")));
}

private void notifyRemovedIssue(String issueId) {
listeners.forEach(c -> c.onRemovedIssue(pr, new Issue(issueId, "")));
private void notifyRemovedIssue(String issueId, Path scratchPath) {
listeners.forEach(c -> c.onRemovedIssue(pr, scratchPath.resolve(c.name()), new Issue(issueId, "")));
}

private void notifyNewPr(PullRequest pr) {
listeners.forEach(c -> c.onNewPullRequest(pr));
private void notifyNewPr(PullRequest pr, Path scratchPath) {
listeners.forEach(c -> c.onNewPullRequest(pr, scratchPath.resolve(c.name())));
}

private void notifyIntegratedPr(PullRequest pr, Hash hash) {
listeners.forEach(c -> c.onIntegratedPullRequest(pr, hash));
private void notifyIntegratedPr(PullRequest pr, Hash hash, Path scratchPath) {
listeners.forEach(c -> c.onIntegratedPullRequest(pr, scratchPath.resolve(c.name()), hash));
}

private void notifyHeadChange(PullRequest pr, Hash oldHead) {
listeners.forEach(c -> c.onHeadChange(pr, oldHead));
private void notifyHeadChange(PullRequest pr, Hash oldHead, Path scratchPath) {
listeners.forEach(c -> c.onHeadChange(pr, scratchPath.resolve(c.name()), oldHead));
}

private void notifyStateChange(org.openjdk.skara.issuetracker.Issue.State oldState) {
listeners.forEach(c -> c.onStateChange(pr, oldState));
private void notifyStateChange(org.openjdk.skara.issuetracker.Issue.State oldState, Path scratchPath) {
listeners.forEach(c -> c.onStateChange(pr, scratchPath.resolve(c.name()), oldState));
}

@Override
public Collection<WorkItem> run(Path scratchPath) {
var historyPath = scratchPath.resolve("notify").resolve("history");
var listenerScratchPath = scratchPath.resolve("notify").resolve("listener");
var storage = prStateStorageBuilder
.serializer(this::serializePrState)
.deserializer(this::deserializePrState)
@@ -235,26 +236,26 @@ public Collection<WorkItem> run(Path scratchPath) {
var storedIssues = storedState.get().issueIds();
storedIssues.stream()
.filter(issue -> !issues.contains(issue))
.forEach(this::notifyRemovedIssue);
.forEach(issue -> notifyRemovedIssue(issue, listenerScratchPath));
issues.stream()
.filter(issue -> !storedIssues.contains(issue))
.forEach(this::notifyNewIssue);
.forEach(issue -> notifyNewIssue(issue, listenerScratchPath));

if (!storedState.get().head().equals(state.head())) {
notifyHeadChange(pr, storedState.get().head());
notifyHeadChange(pr, storedState.get().head(), listenerScratchPath);
}
var storedCommit = storedState.get().commitId();
if (storedCommit.isEmpty() && state.commitId().isPresent()) {
notifyIntegratedPr(pr, state.commitId().get());
notifyIntegratedPr(pr, state.commitId().get(), listenerScratchPath);
}
if (!storedState.get().state().equals(state.state())) {
notifyStateChange(storedState.get().state());
notifyStateChange(storedState.get().state(), scratchPath);
}
} else {
notifyNewPr(pr);
issues.forEach(this::notifyNewIssue);
notifyNewPr(pr, listenerScratchPath);
issues.forEach(issue -> notifyNewIssue(issue, listenerScratchPath));
if (state.commitId().isPresent()) {
notifyIntegratedPr(pr, state.commitId().get());
notifyIntegratedPr(pr, state.commitId().get(), listenerScratchPath);
}
}

@@ -26,16 +26,17 @@
import org.openjdk.skara.vcs.*;
import org.openjdk.skara.vcs.openjdk.OpenJDKTag;

import java.nio.file.Path;
import java.util.List;

public interface RepositoryListener {
default void onNewCommits(HostedRepository repository, Repository localRepository, List<Commit> commits, Branch branch) throws NonRetriableException {
default void onNewCommits(HostedRepository repository, Repository localRepository, Path scratchPath, List<Commit> commits, Branch branch) throws NonRetriableException {
}
default void onNewOpenJDKTagCommits(HostedRepository repository, Repository localRepository, List<Commit> commits, OpenJDKTag tag, Tag.Annotated annotated) throws NonRetriableException {
default void onNewOpenJDKTagCommits(HostedRepository repository, Repository localRepository, Path scratchPath, List<Commit> commits, OpenJDKTag tag, Tag.Annotated annotated) throws NonRetriableException {
}
default void onNewTagCommit(HostedRepository repository, Repository localRepository, Commit commit, Tag tag, Tag.Annotated annotation) throws NonRetriableException {
default void onNewTagCommit(HostedRepository repository, Repository localRepository, Path scratchPath, Commit commit, Tag tag, Tag.Annotated annotation) throws NonRetriableException {
}
default void onNewBranch(HostedRepository repository, Repository localRepository, List<Commit> commits, Branch parent, Branch branch) throws NonRetriableException {
default void onNewBranch(HostedRepository repository, Repository localRepository, Path scratchPath, List<Commit> commits, Branch parent, Branch branch) throws NonRetriableException {
}
String name();
}
@@ -53,7 +53,7 @@ public class RepositoryWorkItem implements WorkItem {
this.listeners = listeners;
}

private void handleNewRef(Repository localRepo, Reference ref, Collection<Reference> allRefs, RepositoryListener listener) throws NonRetriableException {
private void handleNewRef(Repository localRepo, Reference ref, Collection<Reference> allRefs, RepositoryListener listener, Path scratchPath) throws NonRetriableException {
// Figure out the best parent ref
var candidates = new HashSet<>(allRefs);
candidates.remove(ref);
@@ -84,15 +84,15 @@ private void handleNewRef(Repository localRepo, Reference ref, Collection<Refere
}
var branch = new Branch(ref.name());
var parent = new Branch(bestParent.getKey().name());
listener.onNewBranch(repository, localRepo, bestParentCommits, parent, branch);
listener.onNewBranch(repository, localRepo, scratchPath, bestParentCommits, parent, branch);
}

private void handleUpdatedRef(Repository localRepo, Reference ref, List<Commit> commits, RepositoryListener listener) throws NonRetriableException {
private void handleUpdatedRef(Repository localRepo, Reference ref, List<Commit> commits, RepositoryListener listener, Path scratchPath) throws NonRetriableException {
var branch = new Branch(ref.name());
listener.onNewCommits(repository, localRepo, commits, branch);
listener.onNewCommits(repository, localRepo, scratchPath, commits, branch);
}

private List<Throwable> handleRef(Repository localRepo, UpdateHistory history, Reference ref, Collection<Reference> allRefs) throws IOException {
private List<Throwable> handleRef(Repository localRepo, UpdateHistory history, Reference ref, Collection<Reference> allRefs, Path scratchPath) throws IOException {
var errors = new ArrayList<Throwable>();
var branch = new Branch(ref.name());
for (var listener : listeners) {
@@ -101,7 +101,7 @@ private List<Throwable> handleRef(Repository localRepo, UpdateHistory history, R
log.warning("No previous history found for branch '" + branch + "' and listener '" + listener.name() + " - resetting mark");
history.setBranchHash(branch, listener.name(), ref.hash());
try {
handleNewRef(localRepo, ref, allRefs, listener);
handleNewRef(localRepo, ref, allRefs, listener, scratchPath.resolve(listener.name()));
} catch (NonRetriableException e) {
errors.add(e.cause());
} catch (RuntimeException e) {
@@ -124,7 +124,7 @@ private List<Throwable> handleRef(Repository localRepo, UpdateHistory history, R
var commits = localRepo.commits(lastHash.get() + ".." + ref.hash(), true).asList();
history.setBranchHash(branch, listener.name(), ref.hash());
try {
handleUpdatedRef(localRepo, ref, commits, listener);
handleUpdatedRef(localRepo, ref, commits, listener, scratchPath.resolve(listener.name()));
} catch (NonRetriableException e) {
log.log(Level.INFO, "Non retriable exception occurred", e);
errors.add(e.cause());
@@ -153,7 +153,7 @@ private Optional<OpenJDKTag> existingPrevious(OpenJDKTag tag, Set<OpenJDKTag> al
}
}

private List<Throwable> handleTags(Repository localRepo, UpdateHistory history, RepositoryListener listener) throws IOException {
private List<Throwable> handleTags(Repository localRepo, UpdateHistory history, RepositoryListener listener, Path scratchPath) throws IOException {
var errors = new ArrayList<Throwable>();
var tags = localRepo.tags();
var newTags = tags.stream()
@@ -210,7 +210,7 @@ private List<Throwable> handleTags(Repository localRepo, UpdateHistory history,

history.addTags(List.of(tag.tag()), listener.name());
try {
listener.onNewOpenJDKTagCommits(repository, localRepo, commits, tag, annotation.orElse(null));
listener.onNewOpenJDKTagCommits(repository, localRepo, scratchPath, commits, tag, annotation.orElse(null));
} catch (NonRetriableException e) {
errors.add(e.cause());
} catch (RuntimeException e) {
@@ -232,7 +232,7 @@ private List<Throwable> handleTags(Repository localRepo, UpdateHistory history,

history.addTags(List.of(tag), listener.name());
try {
listener.onNewTagCommit(repository, localRepo, commit.get(), tag, annotation.orElse(null));
listener.onNewTagCommit(repository, localRepo, scratchPath, commit.get(), tag, annotation.orElse(null));
} catch (NonRetriableException e) {
errors.add(e.cause());
} catch (RuntimeException e) {
@@ -260,6 +260,7 @@ public boolean concurrentWith(WorkItem other) {
public Collection<WorkItem> run(Path scratchPath) {
var historyPath = scratchPath.resolve("notify").resolve("history");
var repositoryPool = new HostedRepositoryPool(storagePath.resolve("seeds"));
var notifierScratchPath = scratchPath.resolve("notify").resolve("notifier");

try {
var localRepo = repositoryPool.materializeBare(repository, scratchPath.resolve("notify").resolve("repowi").resolve(repository.name()));
@@ -273,7 +274,7 @@ public Collection<WorkItem> run(Path scratchPath) {
var errors = new ArrayList<Throwable>();

for (var listener : listeners) {
errors.addAll(handleTags(localRepo, history, listener));
errors.addAll(handleTags(localRepo, history, listener, notifierScratchPath.resolve(listener.name())));
}

boolean hasBranchHistory = !history.isEmpty();
@@ -285,7 +286,7 @@ public Collection<WorkItem> run(Path scratchPath) {
history.setBranchHash(new Branch(ref.name()), listener.name(), ref.hash());
}
} else {
errors.addAll(handleRef(localRepo, history, ref, knownRefs));
errors.addAll(handleRef(localRepo, history, ref, knownRefs, scratchPath));
}
}
if (!errors.isEmpty()) {
@@ -28,6 +28,7 @@
import org.openjdk.skara.vcs.*;
import org.openjdk.skara.vcs.openjdk.CommitMessageParsers;

import java.nio.file.Path;
import java.util.*;
import java.util.stream.Collectors;

@@ -54,7 +55,7 @@ public void attachTo(Emitter e) {
}

@Override
public void onIntegratedPullRequest(PullRequest pr, Hash hash) {
public void onIntegratedPullRequest(PullRequest pr, Path scratchPath, Hash hash) {
var repository = pr.repository();
var commit = repository.commit(hash).orElseThrow(() ->
new IllegalStateException("Integrated commit " + hash +
@@ -78,4 +79,9 @@ public void onIntegratedPullRequest(PullRequest pr, Hash hash) {
}
repository.addCommitComment(hash, String.join("\n", comment));
}

@Override
public String name() {
return "commitcomment";
}
}
@@ -0,0 +1,84 @@
/*
* Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package org.openjdk.skara.bots.notify.issue;

import org.openjdk.skara.census.*;
import org.openjdk.skara.forge.HostedRepository;
import org.openjdk.skara.vcs.Repository;

import java.io.*;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Optional;

class CensusInstance {
private final Namespace namespace;

private CensusInstance(Namespace namespace) {
this.namespace = namespace;
}

private static Repository initialize(HostedRepository repo, String ref, Path folder) {
try {
return Repository.materialize(folder, repo.url(), "+" + ref + ":" + "issue_census_" + repo.name());
} catch (IOException e) {
throw new RuntimeException("Failed to retrieve census to " + folder, e);
}
}

private static Namespace namespace(Census census, String hostNamespace) {
var namespace = census.namespace(hostNamespace);
if (namespace == null) {
throw new RuntimeException("Namespace not found in census: " + hostNamespace);
}

return namespace;
}

static CensusInstance create(HostedRepository censusRepo, String censusRef, Path folder, String namespace) {
var repoName = censusRepo.url().getHost() + "/" + censusRepo.name();
var repoFolder = folder.resolve(URLEncoder.encode(repoName, StandardCharsets.UTF_8));
try {
var localRepo = Repository.get(repoFolder)
.or(() -> Optional.of(initialize(censusRepo, censusRef, repoFolder)))
.orElseThrow();
var hash = localRepo.fetch(censusRepo.url(), censusRef, false);
localRepo.checkout(hash, true);
} catch (IOException e) {
initialize(censusRepo, censusRef, repoFolder);
}

try {
var census = Census.parse(repoFolder);
var ns = namespace(census, namespace);
return new CensusInstance(ns);
} catch (IOException e) {
throw new UncheckedIOException("Cannot parse census at " + repoFolder, e);
}
}

Namespace namespace() {
return namespace;
}
}

1 comment on commit b8f1016

@openjdk-notifier
Copy link

@openjdk-notifier openjdk-notifier bot commented on b8f1016 Apr 20, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.