From e23b2482888691a5524ee5f87fbbe0229d664dc4 Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Tue, 22 Oct 2019 09:21:34 +0000 Subject: [PATCH 01/54] Add submodule and subrepo support Reviewed-by: rwestberg --- .../openjdk/skara/jcheck/TestRepository.java | 7 +++ .../openjdk/skara/vcs/ReadOnlyRepository.java | 1 + .../org/openjdk/skara/vcs/Repository.java | 1 + .../java/org/openjdk/skara/vcs/Submodule.java | 54 +++++++++++++++++ .../openjdk/skara/vcs/git/GitRepository.java | 59 +++++++++++++++++++ .../openjdk/skara/vcs/hg/HgRepository.java | 44 ++++++++++++++ .../openjdk/skara/vcs/RepositoryTests.java | 46 +++++++++++++++ 7 files changed, 212 insertions(+) create mode 100644 vcs/src/main/java/org/openjdk/skara/vcs/Submodule.java diff --git a/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java b/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java index 12f89ea0b..0ee1af6aa 100644 --- a/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java +++ b/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java @@ -232,4 +232,11 @@ public List remoteBranches(String remote) throws IOException { public List remotes() throws IOException { return null; } + + public void addSubmodule(String pullPath, Path path) throws IOException { + } + + public List submodules() throws IOException { + return null; + } } diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/ReadOnlyRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/ReadOnlyRepository.java index 6f79770fa..3490f7673 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/ReadOnlyRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/ReadOnlyRepository.java @@ -86,6 +86,7 @@ default List files(Hash h, Path... paths) throws IOException { Optional upstreamFor(Branch branch) throws IOException; List remoteBranches(String remote) throws IOException; List remotes() throws IOException; + List submodules() throws IOException; static Optional get(Path p) throws IOException { return Repository.get(p).map(r -> r); diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/Repository.java b/vcs/src/main/java/org/openjdk/skara/vcs/Repository.java index 3a9d07d61..93a589d46 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/Repository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/Repository.java @@ -107,6 +107,7 @@ Hash amend(String message, default void setPaths(String remote, String pullPath) throws IOException { setPaths(remote, pullPath, null); } + void addSubmodule(String pullPath, Path path) throws IOException; default void push(Hash hash, URI uri, String ref) throws IOException { push(hash, uri, ref, false); diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/Submodule.java b/vcs/src/main/java/org/openjdk/skara/vcs/Submodule.java new file mode 100644 index 000000000..102475180 --- /dev/null +++ b/vcs/src/main/java/org/openjdk/skara/vcs/Submodule.java @@ -0,0 +1,54 @@ +package org.openjdk.skara.vcs; + +import java.nio.file.Path; +import java.util.Objects; + +public class Submodule { + private final Hash hash; + private final Path path; + private final String pullPath; + + public Submodule(Hash hash, Path path, String pullPath) { + this.hash = hash; + this.path = path; + this.pullPath = pullPath; + } + + public Hash hash() { + return hash; + } + + public Path path() { + return path; + } + + public String pullPath() { + return pullPath; + } + + @Override + public String toString() { + return pullPath + " " + hash + " " + path; + } + + @Override + public int hashCode() { + return Objects.hash(hash, path, pullPath); + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + + if (!(other instanceof Submodule)) { + return false; + } + + var o = (Submodule) other; + return Objects.equals(hash, o.hash) && + Objects.equals(path, o.path) && + Objects.equals(pullPath, o.pullPath); + } +} diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java index a4fd308d3..776574188 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java @@ -1089,4 +1089,63 @@ public List remotes() throws IOException { } return remotes; } + + @Override + public void addSubmodule(String pullPath, Path path) throws IOException { + try (var p = capture("git", "submodule", "add", pullPath, path.toString())) { + await(p); + } + } + + @Override + public List submodules() throws IOException { + var gitModules = root().resolve(".gitmodules"); + if (!Files.exists(gitModules)) { + return List.of(); + } + + var urls = new HashMap(); + var paths = new HashMap(); + try (var p = capture("git", "config", "--file", gitModules.toAbsolutePath().toString(), + "--list")) { + for (var line : await(p).stdout()) { + if (line.startsWith("submodule.")) { + line = line.substring("submodule.".length()); + var parts = line.split("="); + var nameAndProperty = parts[0].split("\\."); + var name = nameAndProperty[0]; + var prop = nameAndProperty[1]; + var value = parts[1]; + if (prop.equals("path")) { + paths.put(name, value); + } else if (prop.equals("url")) { + urls.put(name, value); + } else { + throw new IOException("Unexpected submodule property: " + prop); + } + } + } + } + + var hashes = new HashMap(); + try (var p = capture("git", "submodule", "status")) { + for (var line : await(p).stdout()) { + var parts = line.substring(1).split(" "); + var hash = parts[0]; + var path = parts[1]; + hashes.put(path, hash); + } + } + + var modules = new ArrayList(); + for (var name : paths.keySet()) { + var url = urls.get(name); + var path = paths.get(name); + var hash = hashes.get(path); + + modules.add(new Submodule(new Hash(hash), Path.of(path), url)); + } + + return modules; + } } diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java index 8f25b5ac5..1ed12e3b3 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java @@ -1126,4 +1126,48 @@ public List remotes() throws IOException { } return remotes; } + + @Override + public void addSubmodule(String pullPath, Path path) throws IOException { + var uri = Files.exists(Path.of(pullPath)) ? Path.of(pullPath).toUri().toString() : pullPath; + HgRepository.clone(URI.create(uri), root().resolve(path).toAbsolutePath(), false); + var hgSub = root().resolve(".hgsub"); + Files.writeString(hgSub, path.toString() + " = " + pullPath + "\n", + StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE); + add(List.of(hgSub)); + } + + @Override + public List submodules() throws IOException { + var hgSub = root().resolve(".hgsub"); + var hgSubState = root().resolve(".hgsubstate"); + if (!(Files.exists(hgSub) && Files.exists(hgSubState))) { + return List.of(); + } + + var urls = new HashMap(); + for (var line : Files.readAllLines(hgSub)) { + var parts = line.split("="); + var path = parts[0].trim(); + var url = parts[1].trim(); + urls.put(path, url); + } + + var hashes = new HashMap(); + for (var line : Files.readAllLines(hgSubState)) { + var parts = line.split(" "); + var hash = parts[0]; + var path = parts[1]; + hashes.put(path, hash); + } + + var modules = new ArrayList(); + for (var path : urls.keySet()) { + var url = urls.get(path); + var hash = hashes.get(path); + modules.add(new Submodule(new Hash(hash), Path.of(path), url)); + } + + return modules; + } } diff --git a/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java b/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java index db105fecd..0f875d035 100644 --- a/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java +++ b/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java @@ -1848,4 +1848,50 @@ void testRemoteBranches(VCS vcs) throws IOException { assertEquals(upstream.defaultBranch().name(), ref.name()); } } + + @ParameterizedTest + @EnumSource(VCS.class) + void testSubmodulesOnEmptyRepo(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory()) { + var repo = Repository.init(dir.path(), vcs); + assertEquals(List.of(), repo.submodules()); + } + } + + @ParameterizedTest + @EnumSource(VCS.class) + void testSubmodulesOnRepoWithNoSubmodules(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory()) { + var repo = Repository.init(dir.path().resolve("repo"), vcs); + var readme = repo.root().resolve("README"); + Files.writeString(readme, "Hello\n"); + repo.add(readme); + repo.commit("Added README", "duke", "duke@openjdk.org"); + assertEquals(List.of(), repo.submodules()); + } + } + + @ParameterizedTest + @EnumSource(VCS.class) + void testSubmodulesOnRepoWithSubmodule(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory()) { + var submodule = Repository.init(dir.path().resolve("submodule"), vcs); + var readme = submodule.root().resolve("README"); + Files.writeString(readme, "Hello\n"); + submodule.add(readme); + var head = submodule.commit("Added README", "duke", "duke@openjdk.org"); + + var repo = Repository.init(dir.path().resolve("repo"), vcs); + var pullPath = submodule.root().toAbsolutePath().toString(); + repo.addSubmodule(pullPath, Path.of("sub")); + repo.commit("Added submodule", "duke", "duke@openjdk.org"); + + var submodules = repo.submodules(); + assertEquals(1, submodules.size()); + var module = submodules.get(0); + assertEquals(Path.of("sub"), module.path()); + assertEquals(head, module.hash()); + assertEquals(pullPath, module.pullPath()); + } + } } From cc7fa4f729aa19375ac709d601d801d72ed75820 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Tue, 22 Oct 2019 10:54:13 +0000 Subject: [PATCH 02/54] 135: Cannot report more than 50 check errors on GitHub Reviewed-by: kcr, ehelin --- .../org/openjdk/skara/bots/pr/CheckTests.java | 35 +++++++++++++++++++ .../skara/forge/GitHubPullRequest.java | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/CheckTests.java b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/CheckTests.java index 94b539820..c4de6b308 100644 --- a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/CheckTests.java +++ b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/CheckTests.java @@ -1008,4 +1008,39 @@ void draft(TestInfo testInfo) throws IOException { assertFalse(pr.labels().contains("ready")); } } + + @Test + void excessiveFailures(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var reviewer = credentials.getHostedRepository(); + + var censusBuilder = credentials.getCensusBuilder() + .addAuthor(author.forge().currentUser().id()) + .addReviewer(reviewer.forge().currentUser().id()); + var checkBot = new PullRequestBot(author, censusBuilder.build(), "master"); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR containing more errors than at least GitHub can handle in a check + var badContent = "\tline \n".repeat(200); + var editHash = CheckableRepository.appendAndCommit(localRepo, badContent); + localRepo.push(editHash, author.url(), "refs/heads/edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", + "This is a pull request", true); + + // Check the status + TestBotRunner.runPeriodicItems(checkBot); + + // Verify that the check failed + var checks = pr.checks(editHash); + assertEquals(1, checks.size()); + var check = checks.get("jcheck"); + assertEquals(CheckStatus.FAILURE, check.status()); + } + } } diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitHubPullRequest.java b/forge/src/main/java/org/openjdk/skara/forge/GitHubPullRequest.java index fc3a6644f..bb9b6e0b2 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitHubPullRequest.java +++ b/forge/src/main/java/org/openjdk/skara/forge/GitHubPullRequest.java @@ -347,7 +347,7 @@ public void updateCheck(Check check) { outputQuery.put("summary", check.summary().get()); var annotations = JSON.array(); - for (var annotation : check.annotations()) { + for (var annotation : check.annotations().subList(0, Math.min(check.annotations().size(), 50))) { var annotationQuery = JSON.object(); annotationQuery.put("path", annotation.path()); annotationQuery.put("start_line", annotation.startLine()); From a53a2d3964b17aa89bdd12fe44b540c2c5148dac Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Wed, 23 Oct 2019 11:52:03 +0000 Subject: [PATCH 03/54] Add --recurse-submodules to applicable actions in GitRepository Reviewed-by: rwestberg --- .../java/org/openjdk/skara/vcs/git/GitRepository.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java index 776574188..2e4a8bc3d 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java @@ -311,7 +311,7 @@ public void reset(Hash target, boolean hard) throws IOException { @Override public void revert(Hash h) throws IOException { - try (var p = capture("git", "checkout", h.hex(), "--", ".")) { + try (var p = capture("git", "checkout", "--recurse-submodules", h.hex(), "--", ".")) { await(p); } } @@ -345,7 +345,7 @@ public void fetchAll() throws IOException { private void checkout(String ref, boolean force) throws IOException { var cmd = new ArrayList(); - cmd.addAll(List.of("git", "-c", "advice.detachedHead=false", "checkout")); + cmd.addAll(List.of("git", "-c", "advice.detachedHead=false", "checkout", "--recurse-submodules")); if (force) { cmd.add("--force"); } @@ -873,7 +873,7 @@ public static Optional get(Path p) throws IOException { @Override public Repository copyTo(Path destination) throws IOException { - try (var p = capture("git", "clone", root().toString(), destination.toString())) { + try (var p = capture("git", "clone", "--recurse-submodules", root().toString(), destination.toString())) { await(p); } @@ -1009,6 +1009,8 @@ public static Repository clone(URI from, Path to, boolean isBare) throws IOExcep cmd.addAll(List.of("git", "clone")); if (isBare) { cmd.add("--bare"); + } else { + cmd.add("--recurse-submodules"); } cmd.addAll(List.of(from.toString(), to.toString())); try (var p = capture(Path.of("").toAbsolutePath(), cmd)) { From 9cd8fef78545c8fa1576bb882bea1f1f51b469a5 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Thu, 24 Oct 2019 06:58:53 +0000 Subject: [PATCH 04/54] 137: Watchdog test can fail to terminate Reviewed-by: ehelin --- bot/src/main/java/org/openjdk/skara/bot/BotRunner.java | 8 +++++--- .../test/java/org/openjdk/skara/bot/BotRunnerTests.java | 7 +++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bot/src/main/java/org/openjdk/skara/bot/BotRunner.java b/bot/src/main/java/org/openjdk/skara/bot/BotRunner.java index 1cb6d1592..81c2da47f 100644 --- a/bot/src/main/java/org/openjdk/skara/bot/BotRunner.java +++ b/bot/src/main/java/org/openjdk/skara/bot/BotRunner.java @@ -177,7 +177,6 @@ private void drain(Duration timeout) throws TimeoutException { } try { Thread.sleep(1); - watchdog(); } catch (InterruptedException e) { log.warning("Exception during queue drain"); log.throwing("BotRunner", "drain", e); @@ -259,7 +258,10 @@ private void processRestRequest(JSONValue request) { } public void run() { - log.info("Starting BotRunner execution, will run forever."); + run(Duration.ofDays(10 * 365)); + } + + public void run(Duration timeout) { log.info("Periodic task interval: " + config.scheduledExecutionPeriod()); log.info("Concurrency: " + config.concurrency()); @@ -280,7 +282,7 @@ public void run() { config.scheduledExecutionPeriod().toMillis(), TimeUnit.MILLISECONDS); try { - executor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + executor.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS); } catch (InterruptedException e) { e.printStackTrace(); } diff --git a/bot/src/test/java/org/openjdk/skara/bot/BotRunnerTests.java b/bot/src/test/java/org/openjdk/skara/bot/BotRunnerTests.java index 9f87cf93f..309bcf215 100644 --- a/bot/src/test/java/org/openjdk/skara/bot/BotRunnerTests.java +++ b/bot/src/test/java/org/openjdk/skara/bot/BotRunnerTests.java @@ -276,12 +276,11 @@ void dontDiscardDifferentBlockedItems() throws TimeoutException { } @Test - @DisabledOnOs(OS.WINDOWS) void watchdogTrigger() throws TimeoutException { var countdownLatch = new CountDownLatch(1); var item = new TestBlockedWorkItem(countdownLatch); var bot = new TestBot(item); - var runner = new BotRunner(config("{ \"runner\": { \"watchdog\": \"PT0.01S\" } }"), List.of(bot)); + var runner = new BotRunner(config("{ \"runner\": { \"watchdog\": \"PT0.01S\", \"interval\": \"PT0.001S\" } }"), List.of(bot)); var errors = new ArrayList(); var log = Logger.getLogger("org.openjdk.skara.bot"); @@ -302,9 +301,9 @@ public void close() throws SecurityException { } }); - assertThrows(TimeoutException.class, () -> runner.runOnce(Duration.ofMillis(100))); + runner.run(Duration.ofMillis(100)); assertTrue(errors.size() > 0); - assertTrue(errors.size() <= 10); + assertTrue(errors.size() <= 100); countdownLatch.countDown(); } } From 51eae328607738480ff576a2eae4a38466df1edf Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Thu, 24 Oct 2019 06:59:53 +0000 Subject: [PATCH 05/54] 138: Check commit notification author email domain Reviewed-by: ehelin --- .../skara/bots/notify/JNotifyBotFactory.java | 3 ++- .../skara/bots/notify/MailingListUpdater.java | 12 ++++++++-- .../skara/bots/notify/UpdaterTests.java | 23 +++++++++++-------- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBotFactory.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBotFactory.java index f4eb8e65b..da51f4462 100644 --- a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBotFactory.java +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBotFactory.java @@ -106,8 +106,9 @@ public List create(BotConfiguration configuration) { .collect(Collectors.toMap(JSONObject.Field::name, field -> field.value().asString())) : Map.of(); var author = mailinglist.contains("author") ? EmailAddress.parse(mailinglist.get("author").asString()) : null; + var allowedDomains = author == null ? Pattern.compile(mailinglist.get("domains").asString()) : null; updaters.add(new MailingListUpdater(listServer.getList(recipient), recipientAddress, sender, author, - includeBranchNames, mode, headers)); + includeBranchNames, mode, headers, allowedDomains)); } } diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/MailingListUpdater.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/MailingListUpdater.java index db3a455a9..65a103a49 100644 --- a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/MailingListUpdater.java +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/MailingListUpdater.java @@ -44,6 +44,7 @@ public class MailingListUpdater implements UpdateConsumer { private final boolean includeBranch; private final Mode mode; private final Map headers; + private final Pattern allowedAuthorDomains; private final Logger log = Logger.getLogger("org.openjdk.skara.bots.notify"); enum Mode { @@ -53,7 +54,7 @@ enum Mode { } MailingListUpdater(MailingList list, EmailAddress recipient, EmailAddress sender, EmailAddress author, - boolean includeBranch, Mode mode, Map headers) { + boolean includeBranch, Mode mode, Map headers, Pattern allowedAuthorDomains) { this.list = list; this.recipient = recipient; this.sender = sender; @@ -61,6 +62,7 @@ enum Mode { this.includeBranch = includeBranch; this.mode = mode; this.headers = headers; + this.allowedAuthorDomains = allowedAuthorDomains; } private String patchToText(Patch patch) { @@ -101,7 +103,13 @@ private String commitToText(HostedRepository repository, Commit commit) { private EmailAddress commitsToAuthor(List commits) { var commit = commits.get(commits.size() - 1); - return EmailAddress.from(commit.committer().name(), commit.committer().email()); + var commitAddress = EmailAddress.from(commit.committer().name(), commit.committer().email()); + var allowedAuthorMatcher = allowedAuthorDomains.matcher(commitAddress.domain()); + if (!allowedAuthorMatcher.matches()) { + return sender; + } else { + return commitAddress; + } } private String commitsToSubject(HostedRepository repository, List commits, Branch branch) { diff --git a/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java b/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java index bc3a8c0c0..d759d5d11 100644 --- a/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java +++ b/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java @@ -186,7 +186,7 @@ void testMailingList(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, MailingListUpdater.Mode.ALL, - Map.of("extra1", "value1", "extra2", "value2")); + Map.of("extra1", "value1", "extra2", "value2"), Pattern.compile("none")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -201,7 +201,7 @@ void testMailingList(TestInfo testInfo) throws IOException { var conversations = mailmanList.conversations(Duration.ofDays(1)); var email = conversations.get(0).first(); assertEquals(sender, email.sender()); - assertEquals(EmailAddress.from("testauthor", "ta@none.none"), email.author()); + assertEquals(sender, email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertTrue(email.subject().contains(": 23456789: More fixes")); assertFalse(email.subject().contains("master")); @@ -237,7 +237,7 @@ void testMailingListMultiple(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, - MailingListUpdater.Mode.ALL, Map.of()); + MailingListUpdater.Mode.ALL, Map.of(), Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -290,7 +290,7 @@ void testMailingListSponsored(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, - MailingListUpdater.Mode.ALL, Map.of()); + MailingListUpdater.Mode.ALL, Map.of(), Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -340,7 +340,7 @@ void testMailingListMultipleBranches(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var author = EmailAddress.from("author", "author@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, author, true, - MailingListUpdater.Mode.ALL, Map.of()); + MailingListUpdater.Mode.ALL, Map.of(), Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master|another"), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -412,7 +412,8 @@ void testMailingListPROnly(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var author = EmailAddress.from("author", "author@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, author, false, - MailingListUpdater.Mode.PR_ONLY, Map.of("extra1", "value1")); + MailingListUpdater.Mode.PR_ONLY, Map.of("extra1", "value1"), + Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -489,7 +490,7 @@ void testMailingListPR(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, - MailingListUpdater.Mode.PR, Map.of()); + MailingListUpdater.Mode.PR, Map.of(), Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -576,9 +577,10 @@ void testMailinglistTag(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, MailingListUpdater.Mode.ALL, - Map.of("extra1", "value1", "extra2", "value2")); + Map.of("extra1", "value1", "extra2", "value2"), + Pattern.compile(".*")); var prOnlyUpdater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, - MailingListUpdater.Mode.PR_ONLY, Map.of()); + MailingListUpdater.Mode.PR_ONLY, Map.of(), Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater, prOnlyUpdater)); @@ -668,7 +670,8 @@ void testMailingListBranch(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, MailingListUpdater.Mode.ALL, - Map.of("extra1", "value1", "extra2", "value2")); + Map.of("extra1", "value1", "extra2", "value2"), + Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master|newbranch."), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history From 6accfe1f09ebdaa55f4a6d6b5f8d1d09c89863a9 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Thu, 24 Oct 2019 12:49:27 +0200 Subject: [PATCH 06/54] Run Windows CI using cmd --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9ea9a0c6..abb9c202b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,3 +25,4 @@ jobs: - uses: actions/checkout@v1 - name: Build and test run: gradlew.bat test --info --stacktrace + shell: cmd From c96d768b276a679314a43dfacd353d8320e6bf17 Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Fri, 25 Oct 2019 13:24:53 +0000 Subject: [PATCH 07/54] Add --recurse-submodules=on-demand to git fetch commands Reviewed-by: rwestberg --- .../main/java/org/openjdk/skara/vcs/git/GitRepository.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java index 2e4a8bc3d..49bbb783c 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java @@ -330,7 +330,7 @@ public Repository reinitialize() throws IOException { @Override public Hash fetch(URI uri, String refspec) throws IOException { - try (var p = capture("git", "fetch", "--tags", uri.toString(), refspec)) { + try (var p = capture("git", "fetch", "--recurse-submodules=on-demand", "--tags", uri.toString(), refspec)) { await(p); return resolve("FETCH_HEAD").get(); } @@ -338,7 +338,7 @@ public Hash fetch(URI uri, String refspec) throws IOException { @Override public void fetchAll() throws IOException { - try (var p = capture("git", "fetch", "--tags", "--prune", "--prune-tags", "--all")) { + try (var p = capture("git", "fetch", "--recurse-submodules=on-demand", "--tags", "--prune", "--prune-tags", "--all")) { await(p); } } From feb10a0cb62fbaedd068345ee00f8a7b09a3ee4f Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Fri, 25 Oct 2019 13:29:31 +0000 Subject: [PATCH 08/54] 142: The new_line field of a GitLab review comment can be null Reviewed-by: ehelin --- .../java/org/openjdk/skara/forge/GitLabMergeRequest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitLabMergeRequest.java b/forge/src/main/java/org/openjdk/skara/forge/GitLabMergeRequest.java index a13900f86..4e461079a 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitLabMergeRequest.java +++ b/forge/src/main/java/org/openjdk/skara/forge/GitLabMergeRequest.java @@ -160,11 +160,15 @@ public void addReview(Review.Verdict verdict, String body) { } private ReviewComment parseReviewComment(String discussionId, ReviewComment parent, JSONObject note) { + var line = note.get("position").get("new_line").isNull() ? + note.get("position").get("old_line").asInt() : + note.get("position").get("new_line").asInt(); + var comment = new ReviewComment(parent, discussionId, new Hash(note.get("position").get("head_sha").asString()), note.get("position").get("new_path").asString(), - note.get("position").get("new_line").asInt(), + line, note.get("id").toString(), note.get("body").asString(), new HostUser(note.get("author").get("id").asInt(), From 68620913f65e6859ae5480dc37703b7234f860d3 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Mon, 28 Oct 2019 07:07:31 +0000 Subject: [PATCH 09/54] Create forges from service providers Reviewed-by: ehelin --- .../skara/bot/BotRunnerConfiguration.java | 32 ++--- .../skara/bots/cli/BotLogstashHandler.java | 2 +- .../skara/bots/cli/BotSlackHandler.java | 3 +- .../java/org/openjdk/skara/cli/GitFork.java | 7 +- .../java/org/openjdk/skara/cli/GitPr.java | 22 ++-- forge/build.gradle | 2 + forge/src/main/java/module-info.java | 4 + .../java/org/openjdk/skara/forge/Forge.java | 35 +++++- .../org/openjdk/skara/forge/ForgeFactory.java | 70 ++++------- .../skara/forge/PullRequestUpdateCache.java | 2 + .../forge/{ => github}/GitHubApplication.java | 16 ++- .../forge/github/GitHubForgeFactory.java | 39 ++++++ .../skara/forge/{ => github}/GitHubHost.java | 46 ++++--- .../forge/{ => github}/GitHubPullRequest.java | 112 ++++++++++-------- .../forge/{ => github}/GitHubRepository.java | 6 +- .../forge/{ => github}/PositionMapper.java | 2 +- .../forge/gitlab/GitLabForgeFactory.java | 23 ++++ .../skara/forge/{ => gitlab}/GitLabHost.java | 36 ++++-- .../{ => gitlab}/GitLabMergeRequest.java | 7 +- .../forge/{ => gitlab}/GitLabRepository.java | 7 +- .../org/openjdk/skara/forge/ForgeTests.java | 68 +++++++++++ .../openjdk/skara/forge/GitHubHostTests.java | 82 ------------- .../{ => github}/GitHubApplicationTests.java | 4 +- .../skara/forge/github/GitHubHostTests.java | 76 ++++++++++++ .../{ => github}/PositionMapperTests.java | 3 +- ...rsonalAccessToken.java => Credential.java} | 20 ++-- .../issuetracker/IssueTrackerFactory.java | 4 +- .../openjdk/skara/network/RestRequest.java | 27 +++-- .../openjdk/skara/test/HostCredentials.java | 19 +-- 29 files changed, 488 insertions(+), 288 deletions(-) rename forge/src/main/java/org/openjdk/skara/forge/{ => github}/GitHubApplication.java (95%) create mode 100644 forge/src/main/java/org/openjdk/skara/forge/github/GitHubForgeFactory.java rename forge/src/main/java/org/openjdk/skara/forge/{ => github}/GitHubHost.java (82%) rename forge/src/main/java/org/openjdk/skara/forge/{ => github}/GitHubPullRequest.java (84%) rename forge/src/main/java/org/openjdk/skara/forge/{ => github}/GitHubRepository.java (98%) rename forge/src/main/java/org/openjdk/skara/forge/{ => github}/PositionMapper.java (99%) create mode 100644 forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabForgeFactory.java rename forge/src/main/java/org/openjdk/skara/forge/{ => gitlab}/GitLabHost.java (84%) rename forge/src/main/java/org/openjdk/skara/forge/{ => gitlab}/GitLabMergeRequest.java (99%) rename forge/src/main/java/org/openjdk/skara/forge/{ => gitlab}/GitLabRepository.java (98%) create mode 100644 forge/src/test/java/org/openjdk/skara/forge/ForgeTests.java delete mode 100644 forge/src/test/java/org/openjdk/skara/forge/GitHubHostTests.java rename forge/src/test/java/org/openjdk/skara/forge/{ => github}/GitHubApplicationTests.java (96%) create mode 100644 forge/src/test/java/org/openjdk/skara/forge/github/GitHubHostTests.java rename forge/src/test/java/org/openjdk/skara/forge/{ => github}/PositionMapperTests.java (99%) rename host/src/main/java/org/openjdk/skara/host/{PersonalAccessToken.java => Credential.java} (76%) diff --git a/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java b/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java index 410dfdc00..4bab1da68 100644 --- a/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java +++ b/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java @@ -23,19 +23,19 @@ package org.openjdk.skara.bot; import org.openjdk.skara.forge.*; -import org.openjdk.skara.host.*; +import org.openjdk.skara.host.Credential; import org.openjdk.skara.issuetracker.*; -import org.openjdk.skara.network.URIBuilder; import org.openjdk.skara.json.JSONObject; +import org.openjdk.skara.network.URIBuilder; import org.openjdk.skara.vcs.VCS; import java.io.*; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.time.Duration; import java.util.*; import java.util.logging.Logger; -import java.util.regex.Pattern; public class BotRunnerConfiguration { private final Logger log; @@ -64,8 +64,8 @@ private Map parseRepositoryHosts(JSONObject config, Path cwd) thr if (entry.value().contains("gitlab")) { var gitlab = entry.value().get("gitlab"); var uri = URIBuilder.base(gitlab.get("url").asString()).build(); - var pat = new PersonalAccessToken(gitlab.get("username").asString(), gitlab.get("pat").asString()); - ret.put(entry.name(), ForgeFactory.createGitLabHost(uri, pat)); + var pat = new Credential(gitlab.get("username").asString(), gitlab.get("pat").asString()); + ret.put(entry.name(), Forge.from("gitlab", uri, pat, gitlab.asObject())); } else if (entry.value().contains("github")) { var github = entry.value().get("github"); URI uri; @@ -74,21 +74,21 @@ private Map parseRepositoryHosts(JSONObject config, Path cwd) thr } else { uri = URIBuilder.base("https://github.com/").build(); } - Pattern webUriPattern = null; - String webUriReplacement = null; - if (github.contains("weburl")) { - webUriPattern = Pattern.compile(github.get("weburl").get("pattern").asString()); - webUriReplacement = github.get("weburl").get("replacement").asString(); - } if (github.contains("app")) { var keyFile = cwd.resolve(github.get("app").get("key").asString()); - ret.put(entry.name(), ForgeFactory.createGitHubHost(uri, webUriPattern, webUriReplacement, keyFile.toString(), - github.get("app").get("id").asString(), - github.get("app").get("installation").asString())); + try { + var keyContents = Files.readString(keyFile, StandardCharsets.UTF_8); + var pat = new Credential(github.get("app").get("id").asString() + ";" + + github.get("app").get("installation").asString(), + keyContents); + ret.put(entry.name(), Forge.from("github", uri, pat, github.asObject())); + } catch (IOException e) { + throw new ConfigurationError("Cannot find key file: " + keyFile); + } } else { - var pat = new PersonalAccessToken(github.get("username").asString(), github.get("pat").asString()); - ret.put(entry.name(), ForgeFactory.createGitHubHost(uri, pat)); + var pat = new Credential(github.get("username").asString(), github.get("pat").asString()); + ret.put(entry.name(), Forge.from("github", uri, pat, github.asObject())); } } else { throw new ConfigurationError("Host " + entry.name()); diff --git a/bots/cli/src/main/java/org/openjdk/skara/bots/cli/BotLogstashHandler.java b/bots/cli/src/main/java/org/openjdk/skara/bots/cli/BotLogstashHandler.java index 2e41d7b14..2b49d729e 100644 --- a/bots/cli/src/main/java/org/openjdk/skara/bots/cli/BotLogstashHandler.java +++ b/bots/cli/src/main/java/org/openjdk/skara/bots/cli/BotLogstashHandler.java @@ -88,7 +88,7 @@ private void publishToLogstash(Instant time, Level level, String message, Map host.get().user(u)) .collect(Collectors.toList()); pr.setAssignees(assignees); } @@ -569,7 +575,7 @@ public static void main(String[] args) throws IOException, InterruptedException System.exit(1); } - var remoteRepo = host.repository(projectName(uri)); + var remoteRepo = host.get().repository(projectName(uri)); if (token == null) { GitCredentials.approve(credentials); } @@ -637,7 +643,7 @@ public static void main(String[] args) throws IOException, InterruptedException if (arguments.contains("assignees")) { var usernames = Arrays.asList(arguments.get("assignees").asString().split(",")); var assignees = usernames.stream() - .map(host::user) + .map(u -> host.get().user(u)) .collect(Collectors.toList()); pr.setAssignees(assignees); } @@ -834,7 +840,7 @@ public static void main(String[] args) throws IOException, InterruptedException if (arguments.contains("assignees")) { var usernames = Arrays.asList(arguments.get("assignees").asString().split(",")); var assignees = usernames.stream() - .map(host::user) + .map(u -> host.get().user(u)) .collect(Collectors.toList()); pr.setAssignees(assignees); } diff --git a/forge/build.gradle b/forge/build.gradle index 30d028cef..ba1a42bfb 100644 --- a/forge/build.gradle +++ b/forge/build.gradle @@ -28,6 +28,8 @@ module { requires 'org.junit.jupiter.api' requires 'jdk.httpserver' opens 'org.openjdk.skara.forge' to 'org.junit.platform.commons' + opens 'org.openjdk.skara.forge.github' to 'org.junit.platform.commons' + opens 'org.openjdk.skara.forge.gitlab' to 'org.junit.platform.commons' } } diff --git a/forge/src/main/java/module-info.java b/forge/src/main/java/module-info.java index 243ff71e4..824f6d41c 100644 --- a/forge/src/main/java/module-info.java +++ b/forge/src/main/java/module-info.java @@ -34,4 +34,8 @@ requires java.logging; exports org.openjdk.skara.forge; + + uses org.openjdk.skara.forge.ForgeFactory; + + provides org.openjdk.skara.forge.ForgeFactory with org.openjdk.skara.forge.github.GitHubForgeFactory, org.openjdk.skara.forge.gitlab.GitLabForgeFactory; } diff --git a/forge/src/main/java/org/openjdk/skara/forge/Forge.java b/forge/src/main/java/org/openjdk/skara/forge/Forge.java index fff5464d3..90dbc58bc 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/Forge.java +++ b/forge/src/main/java/org/openjdk/skara/forge/Forge.java @@ -23,17 +23,44 @@ package org.openjdk.skara.forge; import org.openjdk.skara.host.*; +import org.openjdk.skara.json.JSONObject; import java.net.URI; +import java.util.*; +import java.util.stream.Collectors; public interface Forge extends Host { HostedRepository repository(String name); boolean supportsReviewBody(); - static Forge from(URI uri, PersonalAccessToken pat) { - return ForgeFactory.createFromURI(uri, pat); + static Forge from(String name, URI uri, Credential credential, JSONObject configuration) { + var factory = ForgeFactory.getForgeFactories().stream() + .filter(f -> f.name().equals(name)) + .findFirst(); + if (factory.isEmpty()) { + throw new RuntimeException("No forge factory named '" + name + "' found - check module path"); + } + return factory.get().create(uri, credential, configuration); } - static Forge from(URI uri) { - return ForgeFactory.createFromURI(uri, null); + + static Optional from(URI uri, Credential credential, JSONObject configuration) { + var factories = ForgeFactory.getForgeFactories().stream() + .sorted(Comparator.comparing(f -> !uri.getHost().contains(f.name()))) + .collect(Collectors.toList()); + for (var factory : factories) { + var forge = factory.create(uri, credential, configuration); + if (forge.isValid()) { + return Optional.of(forge); + } + } + return Optional.empty(); + } + + static Optional from(URI uri, Credential credential) { + return from(uri, credential, null); + } + + static Optional from(URI uri) { + return from(uri, null); } } diff --git a/forge/src/main/java/org/openjdk/skara/forge/ForgeFactory.java b/forge/src/main/java/org/openjdk/skara/forge/ForgeFactory.java index eae341827..6f9154432 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/ForgeFactory.java +++ b/forge/src/main/java/org/openjdk/skara/forge/ForgeFactory.java @@ -22,56 +22,30 @@ */ package org.openjdk.skara.forge; -import org.openjdk.skara.host.*; +import org.openjdk.skara.host.Credential; +import org.openjdk.skara.json.JSONObject; import java.net.URI; -import java.util.regex.Pattern; - -public class ForgeFactory { - public static Forge createGitHubHost(URI uri, Pattern webUriPattern, String webUriReplacement, String keyFile, String issue, String id) { - var app = new GitHubApplication(keyFile, issue, id); - return new GitHubHost(uri, app, webUriPattern, webUriReplacement); - } - - public static Forge createGitHubHost(URI uri, PersonalAccessToken pat) { - if (pat != null) { - return new GitHubHost(uri, pat); - } else { - return new GitHubHost(uri); - } - } - - public static Forge createGitLabHost(URI uri, PersonalAccessToken pat) { - if (pat != null) { - return new GitLabHost(uri, pat); - } else { - return new GitLabHost(uri); - } +import java.util.*; +import java.util.stream.*; + +public interface ForgeFactory { + + /** + * A user-friendly name for the given forge, used for configuration section naming. Should be lower case. + * @return + */ + String name(); + + /** + * Instantiate an instance of this forge. + * @return + */ + Forge create(URI uri, Credential credential, JSONObject configuration); + + static List getForgeFactories() { + return StreamSupport.stream(ServiceLoader.load(ForgeFactory.class).spliterator(), false) + .collect(Collectors.toList()); } - public static Forge createFromURI(URI uri, PersonalAccessToken pat) throws IllegalArgumentException { - // Short-circuit - if (uri.toString().contains("github")) { - return createGitHubHost(uri, pat); - } else if (uri.toString().contains("gitlab")) { - return createGitLabHost(uri, pat); - } - - try { - var gitLabHost = createGitLabHost(uri, pat); - if (gitLabHost.isValid()) { - return gitLabHost; - } - } catch (RuntimeException e) { - try { - var gitHubHost = createGitHubHost(uri, pat); - if (gitHubHost.isValid()) { - return gitHubHost; - } - } catch (RuntimeException ignored) { - } - } - - throw new IllegalArgumentException("Unable to detect host type from URI: " + uri); - } } diff --git a/forge/src/main/java/org/openjdk/skara/forge/PullRequestUpdateCache.java b/forge/src/main/java/org/openjdk/skara/forge/PullRequestUpdateCache.java index c67c4947b..f9af93680 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/PullRequestUpdateCache.java +++ b/forge/src/main/java/org/openjdk/skara/forge/PullRequestUpdateCache.java @@ -22,6 +22,8 @@ */ package org.openjdk.skara.forge; +import org.openjdk.skara.forge.gitlab.GitLabMergeRequest; + import java.time.ZonedDateTime; import java.util.*; import java.util.logging.Logger; diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitHubApplication.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubApplication.java similarity index 95% rename from forge/src/main/java/org/openjdk/skara/forge/GitHubApplication.java rename to forge/src/main/java/org/openjdk/skara/forge/github/GitHubApplication.java index bb8955ef9..0d6a09e7f 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitHubApplication.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubApplication.java @@ -20,16 +20,15 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.github; -import org.openjdk.skara.network.URIBuilder; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.URIBuilder; import java.io.*; import java.net.URI; import java.net.http.*; import java.nio.charset.StandardCharsets; -import java.nio.file.*; import java.security.*; import java.security.spec.*; import java.time.*; @@ -104,7 +103,7 @@ public GitHubConfigurationError(String message) { } } - public GitHubApplication(String keyFile, String issue, String id) { + public GitHubApplication(String key, String issue, String id) { log = Logger.getLogger("org.openjdk.host.github"); @@ -112,14 +111,13 @@ public GitHubApplication(String keyFile, String issue, String id) { this.issue = issue; this.id = id; - key = loadPkcs8PemFromFile(keyFile); + this.key = loadPkcs8PemFromString(key); jwt = new Token(this::generateJsonWebToken, Duration.ofMinutes(5)); installationToken = new Token(this::generateInstallationToken, Duration.ofMinutes(30)); } - private PrivateKey loadPkcs8PemFromFile(String keyFile) { + private PrivateKey loadPkcs8PemFromString(String pem) { try { - var pem = new String(Files.readAllBytes(Paths.get(keyFile))); var pemPattern = Pattern.compile("^-*BEGIN PRIVATE KEY-*$(.*)^-*END PRIVATE KEY-*", Pattern.DOTALL | Pattern.MULTILINE); var keyString = pemPattern.matcher(pem).replaceFirst("$1"); @@ -127,8 +125,8 @@ private PrivateKey loadPkcs8PemFromFile(String keyFile) { var rawKey = Base64.getMimeDecoder().decode(keyString); var factory = KeyFactory.getInstance("RSA"); return factory.generatePrivate(new PKCS8EncodedKeySpec(rawKey)); - } catch (NoSuchAlgorithmException | InvalidKeySpecException | IOException e) { - throw new GitHubConfigurationError("Unable to load private key (" + keyFile + ": " + e + ")"); + } catch (NoSuchAlgorithmException | InvalidKeySpecException e) { + throw new GitHubConfigurationError("Unable to load private key (" + e + ")"); } } diff --git a/forge/src/main/java/org/openjdk/skara/forge/github/GitHubForgeFactory.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubForgeFactory.java new file mode 100644 index 000000000..42bb59027 --- /dev/null +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubForgeFactory.java @@ -0,0 +1,39 @@ +package org.openjdk.skara.forge.github; + +import org.openjdk.skara.forge.*; +import org.openjdk.skara.host.Credential; +import org.openjdk.skara.json.JSONObject; + +import java.net.URI; +import java.util.regex.Pattern; + +public class GitHubForgeFactory implements ForgeFactory { + @Override + public String name() { + return "github"; + } + + @Override + public Forge create(URI uri, Credential credential, JSONObject configuration) { + Pattern webUriPattern = null; + String webUriReplacement = null; + if (configuration != null && configuration.contains("weburl")) { + webUriPattern = Pattern.compile(configuration.get("weburl").get("pattern").asString()); + webUriReplacement = configuration.get("weburl").get("replacement").asString(); + } + + if (credential != null) { + if (credential.username().contains(";")) { + var separator = credential.username().indexOf(";"); + var id = credential.username().substring(0, separator); + var installation = credential.username().substring(separator + 1); + var app = new GitHubApplication(credential.password(), id, installation); + return new GitHubHost(uri, app, webUriPattern, webUriReplacement); + } else { + return new GitHubHost(uri, credential, webUriPattern, webUriReplacement); + } + } else { + return new GitHubHost(uri, webUriPattern, webUriReplacement); + } + } +} diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitHubHost.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java similarity index 82% rename from forge/src/main/java/org/openjdk/skara/forge/GitHubHost.java rename to forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java index 3414194c9..56c1bc033 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitHubHost.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java @@ -20,15 +20,18 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.github; +import org.openjdk.skara.forge.*; import org.openjdk.skara.host.*; -import org.openjdk.skara.network.*; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; +import java.io.IOException; import java.net.*; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.logging.Logger; import java.util.regex.Pattern; public class GitHubHost implements Forge { @@ -36,9 +39,10 @@ public class GitHubHost implements Forge { private final Pattern webUriPattern; private final String webUriReplacement; private final GitHubApplication application; - private final PersonalAccessToken pat; + private final Credential pat; private final RestRequest request; private HostUser currentUser; + private final Logger log = Logger.getLogger("org.openjdk.skara.forge.github"); public GitHubHost(URI uri, GitHubApplication application, Pattern webUriPattern, String webUriReplacement) { this.uri = uri; @@ -58,10 +62,10 @@ public GitHubHost(URI uri, GitHubApplication application, Pattern webUriPattern, "Accept", "application/vnd.github.antiope-preview+json")); } - public GitHubHost(URI uri, PersonalAccessToken pat) { + public GitHubHost(URI uri, Credential pat, Pattern webUriPattern, String webUriReplacement) { this.uri = uri; - this.webUriPattern = null; - this.webUriReplacement = null; + this.webUriPattern = webUriPattern; + this.webUriReplacement = webUriReplacement; this.pat = pat; this.application = null; @@ -71,13 +75,13 @@ public GitHubHost(URI uri, PersonalAccessToken pat) { .build(); request = new RestRequest(baseApi, () -> Arrays.asList( - "Authorization", "token " + pat.token())); + "Authorization", "token " + pat.password())); } - public GitHubHost(URI uri) { + GitHubHost(URI uri, Pattern webUriPattern, String webUriReplacement) { this.uri = uri; - this.webUriPattern = null; - this.webUriReplacement = null; + this.webUriPattern = webUriPattern; + this.webUriReplacement = webUriReplacement; this.pat = null; this.application = null; @@ -114,7 +118,7 @@ String getInstallationToken() { if (application != null) { return application.getInstallationToken(); } else { - return pat.token(); + return pat.password(); } } @@ -135,10 +139,20 @@ HostUser parseUserObject(JSONValue json) { @Override public boolean isValid() { - var endpoints = request.get("") - .onError(response -> JSON.of()) - .execute(); - return !endpoints.isNull(); + try { + var endpoints = request.get("") + .executeUnparsed(); + var parsed = JSON.parse(endpoints); + if (parsed != null && parsed.contains("current_user_url")) { + return true; + } else { + log.fine("Error during GitHub host validation: unexpected endpoint list: " + endpoints); + return false; + } + } catch (IOException e) { + log.fine("Error during GitHub host validation: " + e); + return false; + } } JSONObject getProjectInfo(String name) { @@ -182,7 +196,7 @@ public HostUser currentUser() { var appName = appDetails.get("name").asString() + "[bot]"; currentUser = user(appName); } else if (pat != null) { - currentUser = user(pat.userName()); + currentUser = user(pat.username()); } else { throw new IllegalStateException("No credentials present"); } diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitHubPullRequest.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java similarity index 84% rename from forge/src/main/java/org/openjdk/skara/forge/GitHubPullRequest.java rename to forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java index bb9b6e0b2..4af1d5120 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitHubPullRequest.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java @@ -20,14 +20,16 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.github; -import org.openjdk.skara.host.*; +import org.openjdk.skara.forge.*; +import org.openjdk.skara.host.HostUser; import org.openjdk.skara.issuetracker.*; -import org.openjdk.skara.network.*; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; import org.openjdk.skara.vcs.Hash; +import java.io.*; import java.net.URI; import java.time.*; import java.time.format.DateTimeFormatter; @@ -135,62 +137,74 @@ private ReviewComment parseReviewComment(ReviewComment parent, JSONObject json, @Override public ReviewComment addReviewComment(Hash base, Hash hash, String path, int line, String body) { - var rawDiff = request.get("pulls/" + json.get("number").toString()) - .header("Accept", "application/vnd.github.v3.diff") - .executeUnparsed(); - var diff = PositionMapper.parse(rawDiff); - - var query = JSON.object() - .put("body", body) - .put("commit_id", hash.hex()) - .put("path", path) - .put("position", diff.lineToPosition(path, line)); - var response = request.post("pulls/" + json.get("number").toString() + "/comments") - .body(query) - .execute(); - return parseReviewComment(null, response.asObject(), diff); + try { + var rawDiff = request.get("pulls/" + json.get("number").toString()) + .header("Accept", "application/vnd.github.v3.diff") + .executeUnparsed(); + var diff = PositionMapper.parse(rawDiff); + + var query = JSON.object() + .put("body", body) + .put("commit_id", hash.hex()) + .put("path", path) + .put("position", diff.lineToPosition(path, line)); + var response = request.post("pulls/" + json.get("number").toString() + "/comments") + .body(query) + .execute(); + return parseReviewComment(null, response.asObject(), diff); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } @Override public ReviewComment addReviewCommentReply(ReviewComment parent, String body) { - var rawDiff = request.get("pulls/" + json.get("number").toString()) - .header("Accept", "application/vnd.github.v3.diff") - .executeUnparsed(); - var diff = PositionMapper.parse(rawDiff); - - var query = JSON.object() - .put("body", body) - .put("in_reply_to", Integer.parseInt(parent.threadId())); - var response = request.post("pulls/" + json.get("number").toString() + "/comments") - .body(query) - .execute(); - return parseReviewComment(parent, response.asObject(), diff); + try { + var rawDiff = request.get("pulls/" + json.get("number").toString()) + .header("Accept", "application/vnd.github.v3.diff") + .executeUnparsed(); + var diff = PositionMapper.parse(rawDiff); + + var query = JSON.object() + .put("body", body) + .put("in_reply_to", Integer.parseInt(parent.threadId())); + var response = request.post("pulls/" + json.get("number").toString() + "/comments") + .body(query) + .execute(); + return parseReviewComment(parent, response.asObject(), diff); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } @Override public List reviewComments() { - var rawDiff = request.get("pulls/" + json.get("number").toString()) - .header("Accept", "application/vnd.github.v3.diff") - .executeUnparsed(); - var diff = PositionMapper.parse(rawDiff); - - var ret = new ArrayList(); - var reviewComments = request.get("pulls/" + json.get("number").toString() + "/comments").execute().stream() - .map(JSONValue::asObject) - .collect(Collectors.toList()); - var idToComment = new HashMap(); - - for (var reviewComment : reviewComments) { - ReviewComment parent = null; - if (reviewComment.contains("in_reply_to_id")) { - parent = idToComment.get(reviewComment.get("in_reply_to_id").toString()); + try { + var rawDiff = request.get("pulls/" + json.get("number").toString()) + .header("Accept", "application/vnd.github.v3.diff") + .executeUnparsed(); + var diff = PositionMapper.parse(rawDiff); + + var ret = new ArrayList(); + var reviewComments = request.get("pulls/" + json.get("number").toString() + "/comments").execute().stream() + .map(JSONValue::asObject) + .collect(Collectors.toList()); + var idToComment = new HashMap(); + + for (var reviewComment : reviewComments) { + ReviewComment parent = null; + if (reviewComment.contains("in_reply_to_id")) { + parent = idToComment.get(reviewComment.get("in_reply_to_id").toString()); + } + var comment = parseReviewComment(parent, reviewComment, diff); + idToComment.put(comment.id(), comment); + ret.add(comment); } - var comment = parseReviewComment(parent, reviewComment, diff); - idToComment.put(comment.id(), comment); - ret.add(comment); - } - return ret; + return ret; + } catch (IOException e) { + throw new UncheckedIOException(e); + } } @Override diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitHubRepository.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubRepository.java similarity index 98% rename from forge/src/main/java/org/openjdk/skara/forge/GitHubRepository.java rename to forge/src/main/java/org/openjdk/skara/forge/github/GitHubRepository.java index 1fe31066f..ff1980fe5 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitHubRepository.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubRepository.java @@ -20,11 +20,11 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.github; -import org.openjdk.skara.issuetracker.Issue; -import org.openjdk.skara.network.*; +import org.openjdk.skara.forge.*; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; import org.openjdk.skara.vcs.*; import java.net.URI; diff --git a/forge/src/main/java/org/openjdk/skara/forge/PositionMapper.java b/forge/src/main/java/org/openjdk/skara/forge/github/PositionMapper.java similarity index 99% rename from forge/src/main/java/org/openjdk/skara/forge/PositionMapper.java rename to forge/src/main/java/org/openjdk/skara/forge/github/PositionMapper.java index 8f6feeacf..dcb29fe32 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/PositionMapper.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/PositionMapper.java @@ -20,7 +20,7 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.github; import java.util.*; import java.util.logging.Logger; diff --git a/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabForgeFactory.java b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabForgeFactory.java new file mode 100644 index 000000000..38fdcedf8 --- /dev/null +++ b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabForgeFactory.java @@ -0,0 +1,23 @@ +package org.openjdk.skara.forge.gitlab; + +import org.openjdk.skara.forge.*; +import org.openjdk.skara.host.Credential; +import org.openjdk.skara.json.JSONObject; + +import java.net.URI; + +public class GitLabForgeFactory implements ForgeFactory { + @Override + public String name() { + return "gitlab"; + } + + @Override + public Forge create(URI uri, Credential credential, JSONObject configuration) { + if (credential != null) { + return new GitLabHost(uri, credential); + } else { + return new GitLabHost(uri); + } + } +} diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitLabHost.java b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabHost.java similarity index 84% rename from forge/src/main/java/org/openjdk/skara/forge/GitLabHost.java rename to forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabHost.java index 9525eea3c..71f450926 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitLabHost.java +++ b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabHost.java @@ -20,32 +20,36 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.gitlab; +import org.openjdk.skara.forge.*; import org.openjdk.skara.host.*; -import org.openjdk.skara.network.*; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; +import java.io.IOException; import java.net.*; import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.logging.Logger; public class GitLabHost implements Forge { private final URI uri; - private final PersonalAccessToken pat; + private final Credential pat; private final RestRequest request; + private final Logger log = Logger.getLogger("org.openjdk.skara.forge.gitlab"); - public GitLabHost(URI uri, PersonalAccessToken pat) { + GitLabHost(URI uri, Credential pat) { this.uri = uri; this.pat = pat; var baseApi = URIBuilder.base(uri) .setPath("/api/v4/") .build(); - request = new RestRequest(baseApi, () -> Arrays.asList("Private-Token", pat.token())); + request = new RestRequest(baseApi, () -> Arrays.asList("Private-Token", pat.password())); } - public GitLabHost(URI uri) { + GitLabHost(URI uri) { this.uri = uri; this.pat = null; @@ -59,16 +63,26 @@ public URI getUri() { return uri; } - public Optional getPat() { + Optional getPat() { return Optional.ofNullable(pat); } @Override public boolean isValid() { - var version = request.get("version") - .onError(r -> JSON.object().put("invalid", true)) - .execute(); - return !version.contains("invalid"); + try { + var version = request.get("version") + .executeUnparsed(); + var parsed = JSON.parse(version); + if (parsed != null && parsed.contains("version")) { + return true; + } else { + log.fine("Error during GitLab host validation: unexpected version: " + version); + return false; + } + } catch (IOException e) { + log.fine("Error during GitLab host validation: " + e); + return false; + } } JSONObject getProjectInfo(String name) { diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitLabMergeRequest.java b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabMergeRequest.java similarity index 99% rename from forge/src/main/java/org/openjdk/skara/forge/GitLabMergeRequest.java rename to forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabMergeRequest.java index 4e461079a..7ef4e8e21 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitLabMergeRequest.java +++ b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabMergeRequest.java @@ -20,12 +20,13 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.gitlab; -import org.openjdk.skara.host.*; +import org.openjdk.skara.forge.*; +import org.openjdk.skara.host.HostUser; import org.openjdk.skara.issuetracker.*; -import org.openjdk.skara.network.*; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; import org.openjdk.skara.vcs.Hash; import java.net.URI; diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitLabRepository.java b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabRepository.java similarity index 98% rename from forge/src/main/java/org/openjdk/skara/forge/GitLabRepository.java rename to forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabRepository.java index fa1c8cf43..553d4e3e1 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitLabRepository.java +++ b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabRepository.java @@ -20,8 +20,9 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.gitlab; +import org.openjdk.skara.forge.*; import org.openjdk.skara.json.*; import org.openjdk.skara.network.*; import org.openjdk.skara.vcs.*; @@ -51,7 +52,7 @@ public GitLabRepository(GitLabHost gitLabHost, String projectName) { .build(); request = gitLabHost.getPat() - .map(pat -> new RestRequest(baseApi, () -> Arrays.asList("Private-Token", pat.token()))) + .map(pat -> new RestRequest(baseApi, () -> Arrays.asList("Private-Token", pat.password()))) .orElseGet(() -> new RestRequest(baseApi)); var urlPattern = URIBuilder.base(gitLabHost.getUri()) @@ -136,7 +137,7 @@ public URI url() { var builder = URIBuilder .base(gitLabHost.getUri()) .setPath("/" + projectName + ".git"); - gitLabHost.getPat().ifPresent(pat -> builder.setAuthentication(pat.userName() + ":" + pat.token())); + gitLabHost.getPat().ifPresent(pat -> builder.setAuthentication(pat.username() + ":" + pat.password())); return builder.build(); } diff --git a/forge/src/test/java/org/openjdk/skara/forge/ForgeTests.java b/forge/src/test/java/org/openjdk/skara/forge/ForgeTests.java new file mode 100644 index 000000000..63d7065eb --- /dev/null +++ b/forge/src/test/java/org/openjdk/skara/forge/ForgeTests.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.forge; + +import org.junit.jupiter.api.Test; +import org.openjdk.skara.host.Credential; +import org.openjdk.skara.json.JSONObject; + +import java.net.URI; +import java.util.*; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ForgeTests { + @Test + void sortTest() { + var allFactories = List.of(new ForgeFactory() { + @Override + public String name() { + return "something"; + } + + @Override + public Forge create(URI uri, Credential credential, JSONObject configuration) { + return null; + } + }, + new ForgeFactory() { + @Override + public String name() { + return "other"; + } + + @Override + public Forge create(URI uri, Credential credential, JSONObject configuration) { + return null; + } + }); + + var sorted = allFactories.stream() + .sorted(Comparator.comparing(f -> !f.name().contains("other"))) + .collect(Collectors.toList()); + + assertEquals("something", allFactories.get(0).name()); + assertEquals("other", sorted.get(0).name()); + } +} diff --git a/forge/src/test/java/org/openjdk/skara/forge/GitHubHostTests.java b/forge/src/test/java/org/openjdk/skara/forge/GitHubHostTests.java deleted file mode 100644 index a94402354..000000000 --- a/forge/src/test/java/org/openjdk/skara/forge/GitHubHostTests.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package org.openjdk.skara.forge; - -import org.openjdk.skara.network.URIBuilder; -import org.openjdk.skara.test.TemporaryDirectory; - -import org.junit.jupiter.api.*; - -import java.io.IOException; -import java.net.*; -import java.nio.charset.StandardCharsets; -import java.nio.file.*; -import java.util.regex.Pattern; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class GitHubHostTests { - private void generateKeyfile(Path path) throws IOException { - // This key was randomly generated for this test only - Files.writeString(path, "-----BEGIN PRIVATE KEY-----\n" + - "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDAFzH+URXAvOoL\n" + - "0NSdIePQTTVsan13c+7D9tAilJAtRcxUjOz2lMZYBzrdsVYGbktfseEvF6o9dyoX\n" + - "X/py6DM0QqBNW/0uEv1ouS44po0VvykHVXrAq0u8E8HHFtr09VQSO/ceXrFd6haQ\n" + - "aCckbdp1TPn1q8w+U2bRkqUji7zzfwll6AaB4dhKZ1v5NFuff1PWmuk2x7b0u2yR\n" + - "uANLHLqmNB4ik7bUTiIyacXeVSZRZRFGwJjd+1WnyiybwV6QbQ0nndw6iaz2wGWt\n" + - "XDif7DJE0axMReUZVKJLqMagS5R5ra6FdlnUPw0nbJMwnDOLk9ofSfne0LTSTD6K\n" + - "/VZ26izbAgMBAAECggEBALF0vDq1reLgo1dHFSQUquFEcpY1yrMP5wQifyVzGb65\n" + - "PIrfpgomZxXrl/Y2XcKTIg7FxcI7moouDDSL9lMxMByXcIAG+14VLQYSDSFIvA3b\n" + - "C4w666wSk2Ss29eQxbaG7aPqweDMmg6osy+1CHQfCDJVapYKoCTz54i0cNrlvSk0\n" + - "FZ3o99uAvAcLtrsqbnXO57NXQVajoSH0bkMZd+TuZqEIX3CzHoNEVhzvqaKedqA6\n" + - "Cd22Y2m6cnW0H10Chko05FtskLYD+jw275LiUtInplBtG3n5/uDIamsOPo9XG8i0\n" + - "a4rxaJYsRqXYqDOEjLi/QCUrYBtJ+gqT/qMOTjAoKAECgYEA/VPdvc03vScjIu4T\n" + - "vNXjXxv81HZPm/IoTYTgvTvrEqErQ/CIwTQJer1XUJ9M43n+XkVZsMKkUIMlwt2+\n" + - "G0wBwYkDUgIXFEJhb170BVgwyZHE+Djr0E7NunbAv/oQu8AfQzk5HZpcQwxVg8w8\n" + - "Vj2ecLb4GK0D9iJ4zLwlsRw2RukCgYEAwh30AG7gq5y9Mj/BusuDvyNZZKjE/pJz\n" + - "HtC7a/OzOyr+Bpr2VBxVDeEFth22bd/a4ohv1QcwNAa2LzelNfQRQURq/vqpDmuj\n" + - "g0ESQavh3i3Tax2LXO582HWueuNL+8Ufyb6WDJDvYuz0F3WBJhxixP3I7VgMhPWV\n" + - "tK/wEEDDwyMCgYEArR3M4NIHDzpZppsv3dIE6ZAEvWSEjrtzk1YFBwyVXkvJd0o/\n" + - "Clj3SWtu6eeS8bkCfYXC/ypkg6i7+2jxa1ILuShanoZTI0Mhtqwa8jQMUxNMmZy8\n" + - "ecQAjzZsDkVjfgqS0quePn6oIiGhpsnBSeYeCkTfUm2Z0XBJQRAqadgvt1ECgYBK\n" + - "FAgzyhxvIUeKT45s7JGAdcr9gPJ8fAL2tY1wqvWxFL0QZD6w5ocG3uLBFyGxWIY9\n" + - "gPe8ghvBHvaTmlav+k5DbAqw95Ngb29c/Y4sBZ4SncZa0FGIy3JVYMOPHgK3OAjj\n" + - "gpncfcr9I5QbB7qbgqWmq3rsKHfOnbHd3G5upWiPpQKBgCaPW2vyT/nfCvfh0z//\n" + - "QSv0//4zy7pDdOolP5ZRsUo5cU4aiv4XgTSglR2jEJyr4bMYCN/+4tnqp0tIUzt1\n" + - "RWJhXLU1dm4QhCTccWMAyQgktn3SB5Ww3+qyLr1klUwkO+rx8kkNjv3rC/u5EzQ9\n" + - "q3DJ9in4wyYBNPVDB5kJom5i\n" + - "-----END PRIVATE KEY-----", StandardCharsets.UTF_8); - } - - @Test - void webUriPatternReplacement() throws IOException, URISyntaxException { - try (var tempFolder = new TemporaryDirectory()) { - var key = tempFolder.path().resolve("key.pem"); - generateKeyfile(key); - var app = new GitHubApplication(key.toString(), "y", "z"); - var host = new GitHubHost(URIBuilder.base("http://www.example.com").build(), - app, Pattern.compile("^(http://www.example.com)/test/(.*)$"), "$1/another/$2"); - assertEquals(new URI("http://www.example.com/another/hello"), host.getWebURI("/test/hello")); - } - } -} diff --git a/forge/src/test/java/org/openjdk/skara/forge/GitHubApplicationTests.java b/forge/src/test/java/org/openjdk/skara/forge/github/GitHubApplicationTests.java similarity index 96% rename from forge/src/test/java/org/openjdk/skara/forge/GitHubApplicationTests.java rename to forge/src/test/java/org/openjdk/skara/forge/github/GitHubApplicationTests.java index c91dcca42..4481e753e 100644 --- a/forge/src/test/java/org/openjdk/skara/forge/GitHubApplicationTests.java +++ b/forge/src/test/java/org/openjdk/skara/forge/github/GitHubApplicationTests.java @@ -20,9 +20,9 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.github; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.Test; import java.time.Duration; diff --git a/forge/src/test/java/org/openjdk/skara/forge/github/GitHubHostTests.java b/forge/src/test/java/org/openjdk/skara/forge/github/GitHubHostTests.java new file mode 100644 index 000000000..e334d413f --- /dev/null +++ b/forge/src/test/java/org/openjdk/skara/forge/github/GitHubHostTests.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.forge.github; + +import org.junit.jupiter.api.Test; +import org.openjdk.skara.network.URIBuilder; +import org.openjdk.skara.test.TemporaryDirectory; + +import java.io.IOException; +import java.net.*; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class GitHubHostTests { + // This key was randomly generated for this test only + private String key = "-----BEGIN PRIVATE KEY-----\n" + + "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDAFzH+URXAvOoL\n" + + "0NSdIePQTTVsan13c+7D9tAilJAtRcxUjOz2lMZYBzrdsVYGbktfseEvF6o9dyoX\n" + + "X/py6DM0QqBNW/0uEv1ouS44po0VvykHVXrAq0u8E8HHFtr09VQSO/ceXrFd6haQ\n" + + "aCckbdp1TPn1q8w+U2bRkqUji7zzfwll6AaB4dhKZ1v5NFuff1PWmuk2x7b0u2yR\n" + + "uANLHLqmNB4ik7bUTiIyacXeVSZRZRFGwJjd+1WnyiybwV6QbQ0nndw6iaz2wGWt\n" + + "XDif7DJE0axMReUZVKJLqMagS5R5ra6FdlnUPw0nbJMwnDOLk9ofSfne0LTSTD6K\n" + + "/VZ26izbAgMBAAECggEBALF0vDq1reLgo1dHFSQUquFEcpY1yrMP5wQifyVzGb65\n" + + "PIrfpgomZxXrl/Y2XcKTIg7FxcI7moouDDSL9lMxMByXcIAG+14VLQYSDSFIvA3b\n" + + "C4w666wSk2Ss29eQxbaG7aPqweDMmg6osy+1CHQfCDJVapYKoCTz54i0cNrlvSk0\n" + + "FZ3o99uAvAcLtrsqbnXO57NXQVajoSH0bkMZd+TuZqEIX3CzHoNEVhzvqaKedqA6\n" + + "Cd22Y2m6cnW0H10Chko05FtskLYD+jw275LiUtInplBtG3n5/uDIamsOPo9XG8i0\n" + + "a4rxaJYsRqXYqDOEjLi/QCUrYBtJ+gqT/qMOTjAoKAECgYEA/VPdvc03vScjIu4T\n" + + "vNXjXxv81HZPm/IoTYTgvTvrEqErQ/CIwTQJer1XUJ9M43n+XkVZsMKkUIMlwt2+\n" + + "G0wBwYkDUgIXFEJhb170BVgwyZHE+Djr0E7NunbAv/oQu8AfQzk5HZpcQwxVg8w8\n" + + "Vj2ecLb4GK0D9iJ4zLwlsRw2RukCgYEAwh30AG7gq5y9Mj/BusuDvyNZZKjE/pJz\n" + + "HtC7a/OzOyr+Bpr2VBxVDeEFth22bd/a4ohv1QcwNAa2LzelNfQRQURq/vqpDmuj\n" + + "g0ESQavh3i3Tax2LXO582HWueuNL+8Ufyb6WDJDvYuz0F3WBJhxixP3I7VgMhPWV\n" + + "tK/wEEDDwyMCgYEArR3M4NIHDzpZppsv3dIE6ZAEvWSEjrtzk1YFBwyVXkvJd0o/\n" + + "Clj3SWtu6eeS8bkCfYXC/ypkg6i7+2jxa1ILuShanoZTI0Mhtqwa8jQMUxNMmZy8\n" + + "ecQAjzZsDkVjfgqS0quePn6oIiGhpsnBSeYeCkTfUm2Z0XBJQRAqadgvt1ECgYBK\n" + + "FAgzyhxvIUeKT45s7JGAdcr9gPJ8fAL2tY1wqvWxFL0QZD6w5ocG3uLBFyGxWIY9\n" + + "gPe8ghvBHvaTmlav+k5DbAqw95Ngb29c/Y4sBZ4SncZa0FGIy3JVYMOPHgK3OAjj\n" + + "gpncfcr9I5QbB7qbgqWmq3rsKHfOnbHd3G5upWiPpQKBgCaPW2vyT/nfCvfh0z//\n" + + "QSv0//4zy7pDdOolP5ZRsUo5cU4aiv4XgTSglR2jEJyr4bMYCN/+4tnqp0tIUzt1\n" + + "RWJhXLU1dm4QhCTccWMAyQgktn3SB5Ww3+qyLr1klUwkO+rx8kkNjv3rC/u5EzQ9\n" + + "q3DJ9in4wyYBNPVDB5kJom5i\n" + + "-----END PRIVATE KEY-----"; + + + @Test + void webUriPatternReplacement() throws IOException, URISyntaxException { + try (var tempFolder = new TemporaryDirectory()) { + var app = new GitHubApplication(key, "y", "z"); + var host = new GitHubHost(URIBuilder.base("http://www.example.com").build(), + app, Pattern.compile("^(http://www.example.com)/test/(.*)$"), "$1/another/$2"); + assertEquals(new URI("http://www.example.com/another/hello"), host.getWebURI("/test/hello")); + } + } +} diff --git a/forge/src/test/java/org/openjdk/skara/forge/PositionMapperTests.java b/forge/src/test/java/org/openjdk/skara/forge/github/PositionMapperTests.java similarity index 99% rename from forge/src/test/java/org/openjdk/skara/forge/PositionMapperTests.java rename to forge/src/test/java/org/openjdk/skara/forge/github/PositionMapperTests.java index 4f1afd325..f912af8ca 100644 --- a/forge/src/test/java/org/openjdk/skara/forge/PositionMapperTests.java +++ b/forge/src/test/java/org/openjdk/skara/forge/github/PositionMapperTests.java @@ -20,9 +20,10 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.github; import org.junit.jupiter.api.*; +import org.openjdk.skara.forge.github.PositionMapper; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/host/src/main/java/org/openjdk/skara/host/PersonalAccessToken.java b/host/src/main/java/org/openjdk/skara/host/Credential.java similarity index 76% rename from host/src/main/java/org/openjdk/skara/host/PersonalAccessToken.java rename to host/src/main/java/org/openjdk/skara/host/Credential.java index 37e1d9b04..254a8fad3 100644 --- a/host/src/main/java/org/openjdk/skara/host/PersonalAccessToken.java +++ b/host/src/main/java/org/openjdk/skara/host/Credential.java @@ -22,20 +22,20 @@ */ package org.openjdk.skara.host; -public class PersonalAccessToken { - private final String userName; - private final String token; +public class Credential { + private final String username; + private final String password; - public PersonalAccessToken(String userName, String token) { - this.userName = userName; - this.token = token; + public Credential(String username, String password) { + this.username = username; + this.password = password; } - public String token() { - return token; + public String password() { + return password; } - public String userName() { - return userName; + public String username() { + return username; } } diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTrackerFactory.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTrackerFactory.java index ef8358fd4..f8ac171af 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTrackerFactory.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTrackerFactory.java @@ -27,8 +27,8 @@ import java.net.URI; public class IssueTrackerFactory { - public static IssueTracker createJiraHost(URI uri, PersonalAccessToken pat) { - if (pat != null) { + public static IssueTracker createJiraHost(URI uri, Credential credential) { + if (credential != null) { throw new RuntimeException("authentication not implemented yet"); } return new JiraHost(uri); diff --git a/network/src/main/java/org/openjdk/skara/network/RestRequest.java b/network/src/main/java/org/openjdk/skara/network/RestRequest.java index 3261563b0..977509bb9 100644 --- a/network/src/main/java/org/openjdk/skara/network/RestRequest.java +++ b/network/src/main/java/org/openjdk/skara/network/RestRequest.java @@ -24,7 +24,7 @@ import org.openjdk.skara.json.*; -import java.io.IOException; +import java.io.*; import java.net.URI; import java.net.http.*; import java.time.Duration; @@ -150,10 +150,14 @@ public QueryBuilder header(String name, String value) { } public JSONValue execute() { - return RestRequest.this.execute(this); + try { + return RestRequest.this.execute(this); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } - public String executeUnparsed() { + public String executeUnparsed() throws IOException { return RestRequest.this.executeUnparsed(this); } } @@ -216,7 +220,7 @@ void setRetryBackoffStep(Duration duration) { retryBackoffStep = duration; } - private HttpResponse sendRequest(HttpRequest request) { + private HttpResponse sendRequest(HttpRequest request) throws IOException { HttpResponse response; var retryCount = 0; @@ -227,14 +231,18 @@ private HttpResponse sendRequest(HttpRequest request) { .build(); response = client.send(request, HttpResponse.BodyHandlers.ofString()); break; - } catch (IOException | InterruptedException e) { + } catch (InterruptedException | IOException e) { if (retryCount < 5) { try { Thread.sleep(retryCount * retryBackoffStep.toMillis()); } catch (InterruptedException ignored) { } } else { - throw new RuntimeException(e); + try { + throw e; + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } } } retryCount++; @@ -297,7 +305,7 @@ private Map parseLink(String link) { .collect(Collectors.toMap(m -> m.group(2), m -> m.group(1))); } - private JSONValue execute(QueryBuilder queryBuilder) { + private JSONValue execute(QueryBuilder queryBuilder) throws IOException { var request = createRequest(queryBuilder.queryType, queryBuilder.endpoint, queryBuilder.composedBody(), queryBuilder.params, queryBuilder.headers); var response = sendRequest(request); @@ -339,10 +347,13 @@ private JSONValue execute(QueryBuilder queryBuilder) { return new JSONArray(ret.stream().flatMap(JSONArray::stream).toArray(JSONValue[]::new)); } - private String executeUnparsed(QueryBuilder queryBuilder) { + private String executeUnparsed(QueryBuilder queryBuilder) throws IOException { var request = createRequest(queryBuilder.queryType, queryBuilder.endpoint, queryBuilder.composedBody(), queryBuilder.params, queryBuilder.headers); var response = sendRequest(request); + if (response.statusCode() >= 400) { + throw new IOException("Bad response: " + response.statusCode()); + } return response.body(); } diff --git a/test/src/main/java/org/openjdk/skara/test/HostCredentials.java b/test/src/main/java/org/openjdk/skara/test/HostCredentials.java index c62fae797..458b2433c 100644 --- a/test/src/main/java/org/openjdk/skara/test/HostCredentials.java +++ b/test/src/main/java/org/openjdk/skara/test/HostCredentials.java @@ -72,12 +72,15 @@ public Forge createRepositoryHost(int userIndex) { var hostUri = URIBuilder.base(config.get("host").asString()).build(); var apps = config.get("apps").asArray(); var key = configDir.resolve(apps.get(userIndex).get("key").asString()); - return ForgeFactory.createGitHubHost(hostUri, - null, - null, - key.toString(), - apps.get(userIndex).get("id").asString(), - apps.get(userIndex).get("installation").asString()); + try { + var keyContents = Files.readString(key, StandardCharsets.UTF_8); + var pat = new Credential(apps.get(userIndex).get("id").asString() + ";" + + apps.get(userIndex).get("installation").asString(), + keyContents); + return Forge.from("github", hostUri, pat, null); + } catch (IOException e) { + throw new RuntimeException("Cannot read private key: " + key); + } } @Override @@ -112,9 +115,9 @@ private static class GitLabCredentials implements Credentials { public Forge createRepositoryHost(int userIndex) { var hostUri = URIBuilder.base(config.get("host").asString()).build(); var users = config.get("users").asArray(); - var pat = new PersonalAccessToken(users.get(userIndex).get("name").asString(), + var pat = new Credential(users.get(userIndex).get("name").asString(), users.get(userIndex).get("pat").asString()); - return ForgeFactory.createGitLabHost(hostUri, pat); + return Forge.from("gitlab", hostUri, pat, null); } @Override From 06a1bcfe299cf80ef34b738ebc6e38c8953b27d0 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Tue, 29 Oct 2019 06:58:51 +0000 Subject: [PATCH 10/54] Support review comments on deleted parts of files on GitLab Reviewed-by: ehelin --- .../forge/gitlab/GitLabMergeRequest.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabMergeRequest.java b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabMergeRequest.java index 7ef4e8e21..5f7ca267b 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabMergeRequest.java +++ b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabMergeRequest.java @@ -161,14 +161,25 @@ public void addReview(Review.Verdict verdict, String body) { } private ReviewComment parseReviewComment(String discussionId, ReviewComment parent, JSONObject note) { - var line = note.get("position").get("new_line").isNull() ? - note.get("position").get("old_line").asInt() : - note.get("position").get("new_line").asInt(); + int line; + String path; + Hash hash; + + // Is the comment on the old or the new version of the file? + if (note.get("position").get("new_line").isNull()) { + line = note.get("position").get("old_line").asInt(); + path = note.get("position").get("old_path").asString(); + hash = new Hash(note.get("position").get("start_sha").asString()); + } else { + line = note.get("position").get("new_line").asInt(); + path = note.get("position").get("new_path").asString(); + hash = new Hash(note.get("position").get("head_sha").asString()); + } var comment = new ReviewComment(parent, discussionId, - new Hash(note.get("position").get("head_sha").asString()), - note.get("position").get("new_path").asString(), + hash, + path, line, note.get("id").toString(), note.get("body").asString(), From f1f33e282ece0892fd7d2087d318787d541d3a55 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Tue, 29 Oct 2019 06:59:52 +0000 Subject: [PATCH 11/54] 72: Retry jcheck if an exception occurs Reviewed-by: ehelin --- .../org/openjdk/skara/bots/pr/CheckRun.java | 12 +++-- .../org/openjdk/skara/bots/pr/CheckTests.java | 54 +++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java index 9f691b5f7..675d04ad8 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java @@ -450,6 +450,7 @@ private void checkStatus() { var checkBuilder = CheckBuilder.create("jcheck", pr.headHash()); checkBuilder.title("Required"); var censusDomain = censusInstance.configuration().census().domain(); + Exception checkException = null; try { // Post check in-progress @@ -513,11 +514,11 @@ private void checkStatus() { } catch (Exception e) { log.throwing("CommitChecker", "checkStatus", e); newLabels.remove("ready"); - var metadata = workItem.getMetadata(pr.title(), pr.body(), pr.comments(), activeReviews, newLabels, censusInstance, pr.targetHash()); - checkBuilder.metadata(metadata); - checkBuilder.title("Exception occurred during jcheck"); + checkBuilder.metadata("invalid"); + checkBuilder.title("Exception occurred during jcheck - the operation will be retried"); checkBuilder.summary(e.getMessage()); checkBuilder.complete(false); + checkException = e; } var check = checkBuilder.build(); pr.updateCheck(check); @@ -533,5 +534,10 @@ private void checkStatus() { pr.removeLabel(oldLabel); } } + + // After updating the PR, rethrow any exception to automatically retry on transient errors + if (checkException != null) { + throw new RuntimeException("Exception during jcheck", checkException); + } } } diff --git a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/CheckTests.java b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/CheckTests.java index c4de6b308..93f4c0c58 100644 --- a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/CheckTests.java +++ b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/CheckTests.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.*; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.util.*; import java.util.regex.Pattern; @@ -1043,4 +1044,57 @@ void excessiveFailures(TestInfo testInfo) throws IOException { assertEquals(CheckStatus.FAILURE, check.status()); } } + + @Test + void retryOnException(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var reviewer = credentials.getHostedRepository(); + + var censusBuilder = credentials.getCensusBuilder() + .addAuthor(author.forge().currentUser().id()) + .addReviewer(reviewer.forge().currentUser().id()); + var checkBot = new PullRequestBot(author, censusBuilder.build(), "master"); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Break the jcheck configuration + var confPath = tempFolder.path().resolve(".jcheck/conf"); + var oldConf = Files.readString(confPath, StandardCharsets.UTF_8); + Files.writeString(confPath, "Hello there!", StandardCharsets.UTF_8); + localRepo.add(confPath); + var editHash = CheckableRepository.appendAndCommit(localRepo, "A change"); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", + "This is a pull request", true); + + // Check the status - should throw every time + assertThrows(RuntimeException.class, () -> TestBotRunner.runPeriodicItems(checkBot)); + assertThrows(RuntimeException.class, () -> TestBotRunner.runPeriodicItems(checkBot)); + assertThrows(RuntimeException.class, () -> TestBotRunner.runPeriodicItems(checkBot)); + + // Verify that the check failed + var checks = pr.checks(editHash); + assertEquals(1, checks.size()); + var check = checks.get("jcheck"); + assertEquals(CheckStatus.FAILURE, check.status()); + + Files.writeString(confPath, oldConf, StandardCharsets.UTF_8); + localRepo.add(confPath); + editHash = CheckableRepository.appendAndCommit(localRepo, "Another change"); + localRepo.push(editHash, author.url(), "edit"); + + TestBotRunner.runPeriodicItems(checkBot); + + // Verify that the check now passes + checks = pr.checks(editHash); + assertEquals(1, checks.size()); + check = checks.get("jcheck"); + assertEquals(CheckStatus.SUCCESS, check.status()); + } + } } From 4ac34b9a5b75dd52838590677818814ce092dbe6 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Tue, 29 Oct 2019 07:00:20 +0000 Subject: [PATCH 12/54] Use a service provider to create issue trackers Reviewed-by: ehelin --- .../skara/bot/BotRunnerConfiguration.java | 2 +- .../org/openjdk/skara/forge/ForgeFactory.java | 2 - issuetracker/src/main/java/module-info.java | 4 ++ .../openjdk/skara/issuetracker/Comment.java | 1 - .../skara/issuetracker/IssueTracker.java | 15 ++++++- .../issuetracker/IssueTrackerFactory.java | 27 ++++++++--- .../issuetracker/{ => jira}/JiraHost.java | 5 ++- .../issuetracker/{ => jira}/JiraIssue.java | 3 +- .../jira/JiraIssueTrackerFactory.java | 45 +++++++++++++++++++ .../issuetracker/{ => jira}/JiraProject.java | 3 +- 10 files changed, 91 insertions(+), 16 deletions(-) rename issuetracker/src/main/java/org/openjdk/skara/issuetracker/{ => jira}/JiraHost.java (95%) rename issuetracker/src/main/java/org/openjdk/skara/issuetracker/{ => jira}/JiraIssue.java (98%) create mode 100644 issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssueTrackerFactory.java rename issuetracker/src/main/java/org/openjdk/skara/issuetracker/{ => jira}/JiraProject.java (97%) diff --git a/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java b/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java index 4bab1da68..51b84aabc 100644 --- a/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java +++ b/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java @@ -109,7 +109,7 @@ private Map parseIssueHosts(JSONObject config, Path cwd) t if (entry.value().contains("jira")) { var jira = entry.value().get("jira"); var uri = URIBuilder.base(jira.get("url").asString()).build(); - ret.put(entry.name(), IssueTrackerFactory.createJiraHost(uri, null)); + ret.put(entry.name(), IssueTracker.from("jira", uri, null, jira.asObject())); } else { throw new ConfigurationError("Host " + entry.name()); } diff --git a/forge/src/main/java/org/openjdk/skara/forge/ForgeFactory.java b/forge/src/main/java/org/openjdk/skara/forge/ForgeFactory.java index 6f9154432..6af275428 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/ForgeFactory.java +++ b/forge/src/main/java/org/openjdk/skara/forge/ForgeFactory.java @@ -30,7 +30,6 @@ import java.util.stream.*; public interface ForgeFactory { - /** * A user-friendly name for the given forge, used for configuration section naming. Should be lower case. * @return @@ -47,5 +46,4 @@ static List getForgeFactories() { return StreamSupport.stream(ServiceLoader.load(ForgeFactory.class).spliterator(), false) .collect(Collectors.toList()); } - } diff --git a/issuetracker/src/main/java/module-info.java b/issuetracker/src/main/java/module-info.java index c9e3274b5..88ca7bf30 100644 --- a/issuetracker/src/main/java/module-info.java +++ b/issuetracker/src/main/java/module-info.java @@ -33,4 +33,8 @@ requires java.logging; exports org.openjdk.skara.issuetracker; + + uses org.openjdk.skara.issuetracker.IssueTrackerFactory; + + provides org.openjdk.skara.issuetracker.IssueTrackerFactory with org.openjdk.skara.issuetracker.jira.JiraIssueTrackerFactory; } diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/Comment.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/Comment.java index 8cf8cd650..3ffed6b2d 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/Comment.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/Comment.java @@ -27,7 +27,6 @@ import java.time.ZonedDateTime; public class Comment { - private final String id; private final String body; private final HostUser author; diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTracker.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTracker.java index e60e9d8e0..0e75f5c40 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTracker.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTracker.java @@ -22,8 +22,21 @@ */ package org.openjdk.skara.issuetracker; -import org.openjdk.skara.host.Host; +import org.openjdk.skara.host.*; +import org.openjdk.skara.json.JSONObject; + +import java.net.URI; public interface IssueTracker extends Host { IssueProject project(String name); + + static IssueTracker from(String name, URI uri, Credential credential, JSONObject configuration) { + var factory = IssueTrackerFactory.getIssueTrackerFactories().stream() + .filter(f -> f.name().equals(name)) + .findFirst(); + if (factory.isEmpty()) { + throw new RuntimeException("No issue tracker factory named '" + name + "' found - check module path"); + } + return factory.get().create(uri, credential, configuration); + } } diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTrackerFactory.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTrackerFactory.java index f8ac171af..c4627012c 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTrackerFactory.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTrackerFactory.java @@ -22,15 +22,28 @@ */ package org.openjdk.skara.issuetracker; -import org.openjdk.skara.host.*; +import org.openjdk.skara.host.Credential; +import org.openjdk.skara.json.JSONObject; import java.net.URI; +import java.util.*; +import java.util.stream.*; -public class IssueTrackerFactory { - public static IssueTracker createJiraHost(URI uri, Credential credential) { - if (credential != null) { - throw new RuntimeException("authentication not implemented yet"); - } - return new JiraHost(uri); +public interface IssueTrackerFactory { + /** + * A user-friendly name for the given issue tracker, used for configuration section naming. Should be lower case. + * @return + */ + String name(); + + /** + * Instantiate an instance of this issue tracker. + * @return + */ + IssueTracker create(URI uri, Credential credential, JSONObject configuration); + + static List getIssueTrackerFactories() { + return StreamSupport.stream(ServiceLoader.load(IssueTrackerFactory.class).spliterator(), false) + .collect(Collectors.toList()); } } diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraHost.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraHost.java similarity index 95% rename from issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraHost.java rename to issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraHost.java index 682a41df5..439e47f30 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraHost.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraHost.java @@ -20,9 +20,10 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.issuetracker; +package org.openjdk.skara.issuetracker.jira; import org.openjdk.skara.host.*; +import org.openjdk.skara.issuetracker.*; import org.openjdk.skara.network.*; import org.openjdk.skara.json.JSON; @@ -32,7 +33,7 @@ public class JiraHost implements IssueTracker { private final URI uri; private final RestRequest request; - public JiraHost(URI uri) { + JiraHost(URI uri) { this.uri = uri; var baseApi = URIBuilder.base(uri) diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraIssue.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssue.java similarity index 98% rename from issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraIssue.java rename to issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssue.java index 76d44a3f6..b916aae4b 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraIssue.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssue.java @@ -20,9 +20,10 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.issuetracker; +package org.openjdk.skara.issuetracker.jira; import org.openjdk.skara.host.*; +import org.openjdk.skara.issuetracker.*; import org.openjdk.skara.network.*; import org.openjdk.skara.json.JSONValue; diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssueTrackerFactory.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssueTrackerFactory.java new file mode 100644 index 000000000..d27a1f195 --- /dev/null +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssueTrackerFactory.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.issuetracker.jira; + +import org.openjdk.skara.host.Credential; +import org.openjdk.skara.issuetracker.*; +import org.openjdk.skara.json.JSONObject; + +import java.net.URI; + +public class JiraIssueTrackerFactory implements IssueTrackerFactory { + @Override + public String name() { + return "jira"; + } + + @Override + public IssueTracker create(URI uri, Credential credential, JSONObject configuration) { + if (credential == null) { + return new JiraHost(uri); + } else { + throw new RuntimeException("authentication not implemented yet"); + } + } +} diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraProject.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java similarity index 97% rename from issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraProject.java rename to issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java index dc9fe137e..17b2fb057 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraProject.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java @@ -20,8 +20,9 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.issuetracker; +package org.openjdk.skara.issuetracker.jira; +import org.openjdk.skara.issuetracker.*; import org.openjdk.skara.json.JSON; import org.openjdk.skara.network.*; From 7984f332540eaa8f6bc49883a511cfb6ccf4441b Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Mon, 4 Nov 2019 15:34:28 +0000 Subject: [PATCH 13/54] Adjust mime encoding Reviewed-by: ehelin --- email/src/main/java/org/openjdk/skara/email/MimeText.java | 2 +- email/src/test/java/org/openjdk/skara/email/MimeTextTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/email/src/main/java/org/openjdk/skara/email/MimeText.java b/email/src/main/java/org/openjdk/skara/email/MimeText.java index 10dcda8e0..34c1fdd0a 100644 --- a/email/src/main/java/org/openjdk/skara/email/MimeText.java +++ b/email/src/main/java/org/openjdk/skara/email/MimeText.java @@ -34,7 +34,7 @@ public class MimeText { public static String encode(String raw) { var quoteMatcher = encodePattern.matcher(raw); - return quoteMatcher.replaceAll(mo -> "=?utf-8?b?" + Base64.getEncoder().encodeToString(String.valueOf(mo.group(1)).getBytes(StandardCharsets.UTF_8)) + "?="); + return quoteMatcher.replaceAll(mo -> "=?UTF-8?B?" + Base64.getEncoder().encodeToString(String.valueOf(mo.group(1)).getBytes(StandardCharsets.UTF_8)) + "?="); } public static String decode(String encoded) { diff --git a/email/src/test/java/org/openjdk/skara/email/MimeTextTests.java b/email/src/test/java/org/openjdk/skara/email/MimeTextTests.java index efb5e0e56..2a45506b5 100644 --- a/email/src/test/java/org/openjdk/skara/email/MimeTextTests.java +++ b/email/src/test/java/org/openjdk/skara/email/MimeTextTests.java @@ -29,7 +29,7 @@ class MimeTextTests { @Test void encode() { - assertEquals("=?utf-8?b?w6XDpMO2?=", MimeText.encode("åäö")); + assertEquals("=?UTF-8?B?w6XDpMO2?=", MimeText.encode("åäö")); } @Test From 17f2684de8b2624452598fc6030c264a58b93f47 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Tue, 5 Nov 2019 10:37:13 +0100 Subject: [PATCH 14/54] Support the "neutral" check conclusion Reviewed-by: ehelin --- .../org/openjdk/skara/forge/github/GitHubPullRequest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java index 4af1d5120..85a42d806 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java @@ -317,6 +317,8 @@ public Map checks(Hash hash) { checkBuilder.complete(true, completedAt); break; case "failure": + // fallthrough + case "neutral": checkBuilder.complete(false, completedAt); break; default: @@ -342,7 +344,7 @@ public Map checks(Hash hash) { @Override public void createCheck(Check check) { - // update and create are currenly identical operations, both do an HTTP + // update and create are currently identical operations, both do an HTTP // POST to the /repos/:owner/:repo/check-runs endpoint. There is an additional // endpoint explicitly for updating check-runs, but that is not currently used. updateCheck(check); From 8c37b4966fc31242b05b3d44773acf60df00fddb Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Tue, 5 Nov 2019 11:24:10 +0100 Subject: [PATCH 15/54] Discard duplicate checks Reviewed-by: ehelin --- .../java/org/openjdk/skara/forge/github/GitHubPullRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java index 85a42d806..ded9dac40 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java @@ -339,7 +339,7 @@ public Map checks(Hash hash) { } return checkBuilder.build(); - })); + }, (a, b) -> b)); } @Override From 0e3e711471fcbd91607fecb6047891b87c38da1a Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Tue, 5 Nov 2019 10:36:46 +0000 Subject: [PATCH 16/54] Implement Jira authentication Reviewed-by: ehelin --- .../skara/issuetracker/jira/JiraHost.java | 45 +++++++- .../skara/issuetracker/jira/JiraIssue.java | 102 +++++++++++++++--- .../jira/JiraIssueTrackerFactory.java | 9 +- .../skara/issuetracker/jira/JiraProject.java | 63 ++++++++++- .../skara/issuetracker/jira/JiraVault.java | 54 ++++++++++ .../skara/issuetracker/IssueTrackerTests.java | 31 +++++- .../openjdk/skara/test/HostCredentials.java | 51 ++++++++- .../org/openjdk/skara/test/IssueData.java | 2 + .../org/openjdk/skara/test/TestIssue.java | 6 +- 9 files changed, 335 insertions(+), 28 deletions(-) create mode 100644 issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraVault.java diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraHost.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraHost.java index 439e47f30..a18ded303 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraHost.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraHost.java @@ -22,12 +22,13 @@ */ package org.openjdk.skara.issuetracker.jira; -import org.openjdk.skara.host.*; +import org.openjdk.skara.host.HostUser; import org.openjdk.skara.issuetracker.*; +import org.openjdk.skara.json.*; import org.openjdk.skara.network.*; -import org.openjdk.skara.json.JSON; import java.net.URI; +import java.util.Arrays; public class JiraHost implements IssueTracker { private final URI uri; @@ -42,6 +43,14 @@ public class JiraHost implements IssueTracker { request = new RestRequest(baseApi); } + JiraHost(URI uri, JiraVault jiraVault) { + this.uri = uri; + var baseApi = URIBuilder.base(uri) + .setPath("/rest/api/2/") + .build(); + request = new RestRequest(baseApi, () -> Arrays.asList("Cookie", jiraVault.getCookie())); + } + URI getUri() { return uri; } @@ -59,18 +68,44 @@ public IssueProject project(String name) { return new JiraProject(this, request, name); } + private JSONObject userData(String name) { + var data = request.get("user") + .param("username", name) + .execute(); + return data.asObject(); + } + @Override public HostUser user(String username) { - throw new RuntimeException("needs authentication; not implemented yet"); + var data = request.get("user") + .param("username", username) + .execute(); + var user = new HostUser(data.get("name").asString(), + data.get("name").asString(), + data.get("displayName").asString()); + return user; } @Override public HostUser currentUser() { - throw new RuntimeException("needs authentication; not implemented yet"); + var data = request.get("myself").execute(); + var user = new HostUser(data.get("name").asString(), + data.get("name").asString(), + data.get("displayName").asString()); + return user; } @Override public boolean isMemberOf(String groupId, HostUser user) { - throw new RuntimeException("not implemented yet"); + var data = request.get("user") + .param("username", user.id()) + .param("expand", "groups") + .execute(); + for (var group : data.get("groups").get("items").asArray()) { + if (group.get("name").asString().equals(groupId)) { + return true; + } + } + return false; } } diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssue.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssue.java index b916aae4b..545ea2ed0 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssue.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssue.java @@ -24,18 +24,22 @@ import org.openjdk.skara.host.*; import org.openjdk.skara.issuetracker.*; +import org.openjdk.skara.json.*; import org.openjdk.skara.network.*; -import org.openjdk.skara.json.JSONValue; import java.net.URI; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.stream.Collectors; public class JiraIssue implements Issue { private final JiraProject jiraProject; private final RestRequest request; private final JSONValue json; + private static final DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + JiraIssue(JiraProject jiraProject, RestRequest request, JSONValue json) { this.jiraProject = jiraProject; this.request = request; @@ -66,7 +70,10 @@ public String title() { @Override public void setTitle(String title) { - throw new RuntimeException("not implemented yet"); + var query = JSON.object() + .put("fields", JSON.object() + .put("summary", title)); + request.put("").body(query).execute(); } @Override @@ -80,52 +87,98 @@ public String body() { @Override public void setBody(String body) { - throw new RuntimeException("not implemented yet"); + var query = JSON.object() + .put("fields", JSON.object() + .put("description", body)); + request.put("").body(query).execute(); + } + + private Comment parseComment(JSONValue json) { + return new Comment(json.get("id").asString(), + json.get("body").asString(), + new HostUser(json.get("author").get("name").asString(), + json.get("author").get("name").asString(), + json.get("author").get("displayName").asString()), + ZonedDateTime.parse(json.get("created").asString(), dateFormat), + ZonedDateTime.parse(json.get("updated").asString(), dateFormat)); } @Override public List comments() { - throw new RuntimeException("not implemented yet"); + var comments = request.get("/comment") + .param("maxResults", "1000") + .execute(); + return comments.get("comments").stream() + .map(this::parseComment) + .collect(Collectors.toList()); } @Override public Comment addComment(String body) { - throw new RuntimeException("not implemented yet"); + var json = request.post("/comment") + .body("body", body) + .execute(); + return parseComment(json); } @Override public Comment updateComment(String id, String body) { - throw new RuntimeException("not implemented yet"); + var json = request.put("/comment/" + id) + .body("body", body) + .execute(); + return parseComment(json); } @Override public ZonedDateTime createdAt() { - return ZonedDateTime.parse(json.get("fields").get("created").asString()); + return ZonedDateTime.parse(json.get("fields").get("created").asString(), dateFormat); } @Override public ZonedDateTime updatedAt() { - return ZonedDateTime.parse(json.get("fields").get("updated").asString()); + return ZonedDateTime.parse(json.get("fields").get("updated").asString(), dateFormat); } @Override public void setState(State state) { - throw new RuntimeException("not implemented yet"); + var transitions = request.get("/transitions").execute(); + var wantedStateName = state == State.CLOSED ? "Closed" : "Open"; + for (var transition : transitions.get("transitions").asArray()) { + if (transition.get("to").get("name").asString().equals(wantedStateName)) { + var query = JSON.object() + .put("transition", JSON.object() + .put("id", transition.get("id").asString())); + request.post("/transitions") + .body(query) + .execute(); + return; + } + } } @Override public void addLabel(String label) { - throw new RuntimeException("not implemented yet"); + var query = JSON.object() + .put("update", JSON.object() + .put("labels", JSON.array().add(JSON.object() + .put("add", label)))); + request.put("").body(query).execute(); } @Override public void removeLabel(String label) { - throw new RuntimeException("not implemented yet"); + var query = JSON.object() + .put("update", JSON.object() + .put("labels", JSON.array().add(JSON.object() + .put("remove", label)))); + request.put("").body(query).execute(); } @Override public List labels() { - throw new RuntimeException("not implemented yet"); + return json.get("fields").get("labels").stream() + .map(JSONValue::asString) + .collect(Collectors.toList()); } @Override @@ -137,11 +190,32 @@ public URI webUrl() { @Override public List assignees() { - throw new RuntimeException("not implemented yet"); + var assignee = json.get("fields").get("assignee"); + if (assignee.isNull()) { + return List.of(); + } + + var user = new HostUser(assignee.get("name").asString(), + assignee.get("name").asString(), + assignee.get("displayName").asString()); + return List.of(user); } @Override public void setAssignees(List assignees) { - throw new RuntimeException("not implemented yet"); + String assignee; + switch (assignees.size()) { + case 0: + assignee = null; + break; + case 1: + assignee = assignees.get(0).id(); + break; + default: + throw new RuntimeException("multiple assignees not supported"); + } + request.put("/assignee") + .body("name", assignee) + .execute(); } } diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssueTrackerFactory.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssueTrackerFactory.java index d27a1f195..c184e7ead 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssueTrackerFactory.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssueTrackerFactory.java @@ -25,6 +25,7 @@ import org.openjdk.skara.host.Credential; import org.openjdk.skara.issuetracker.*; import org.openjdk.skara.json.JSONObject; +import org.openjdk.skara.network.URIBuilder; import java.net.URI; @@ -39,7 +40,13 @@ public IssueTracker create(URI uri, Credential credential, JSONObject configurat if (credential == null) { return new JiraHost(uri); } else { - throw new RuntimeException("authentication not implemented yet"); + if (credential.username().startsWith("https://")) { + var vaultUrl = URIBuilder.base(credential.username()).build(); + var jiraVault = new JiraVault(vaultUrl, credential.password()); + return new JiraHost(uri, jiraVault); + } else { + throw new RuntimeException("basic authentication not implemented yet"); + } } } } diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java index 17b2fb057..991f868ef 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java @@ -23,7 +23,7 @@ package org.openjdk.skara.issuetracker.jira; import org.openjdk.skara.issuetracker.*; -import org.openjdk.skara.json.JSON; +import org.openjdk.skara.json.*; import org.openjdk.skara.network.*; import java.net.URI; @@ -34,12 +34,51 @@ public class JiraProject implements IssueProject { private final String projectName; private final RestRequest request; + private JSONObject projectMetadataCache = null; + JiraProject(JiraHost host, RestRequest request, String projectName) { this.jiraHost = host; this.projectName = projectName; this.request = request; } + private JSONObject project() { + if (projectMetadataCache == null) { + projectMetadataCache = request.get("project/" + projectName).execute().asObject(); + } + return projectMetadataCache; + } + + private Map issueTypes() { + var ret = new HashMap(); + for (var type : project().get("issueTypes").asArray()) { + ret.put(type.get("name").asString(), type.get("id").asString()); + } + return ret; + } + + private Map components() { + var ret = new HashMap(); + for (var type : project().get("components").asArray()) { + ret.put(type.get("name").asString(), type.get("id").asString()); + } + return ret; + } + + private String projectId() { + return project().get("id").asString(); + } + + private String defaultIssueType() { + return issueTypes().values().stream() + .min(Comparator.naturalOrder()).orElseThrow(); + } + + private String defaultComponent() { + return components().values().stream() + .min(Comparator.naturalOrder()).orElseThrow(); + } + @Override public IssueTracker issueTracker() { return jiraHost; @@ -52,7 +91,22 @@ public URI webUrl() { @Override public Issue createIssue(String title, List body) { - throw new RuntimeException("needs authentication; not implemented yet"); + var query = JSON.object() + .put("fields", JSON.object() + .put("project", JSON.object() + .put("id", projectId())) + .put("issuetype", JSON.object() + .put("id", defaultIssueType())) + .put("components", JSON.array() + .add(JSON.object().put("id", defaultComponent()))) + .put("summary", title) + .put("description", String.join("\n", body))); + + var data = request.post("issue") + .body(query) + .execute(); + + return issue(data.get("key").asString()).orElseThrow(); } @Override @@ -60,11 +114,12 @@ public Optional issue(String id) { if (id.indexOf('-') < 0) { id = projectName.toUpperCase() + "-" + id; } - var issue = request.get("issue/" + id) + var issueRequest = request.restrict("issue/" + id); + var issue = issueRequest.get("") .onError(r -> r.statusCode() == 404 ? JSON.object().put("NOT_FOUND", true) : null) .execute(); if (!issue.contains("NOT_FOUND")) { - return Optional.of(new JiraIssue(this, request, issue)); + return Optional.of(new JiraIssue(this, issueRequest, issue)); } else { return Optional.empty(); } diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraVault.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraVault.java new file mode 100644 index 000000000..22fd95da2 --- /dev/null +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraVault.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.issuetracker.jira; + +import org.openjdk.skara.network.RestRequest; + +import java.net.URI; +import java.time.*; +import java.util.Arrays; +import java.util.logging.Logger; + +class JiraVault { + private final RestRequest request; + private final Logger log = Logger.getLogger("org.openjdk.skara.issuetracker.jira"); + + private String cookie; + private Instant expires; + + JiraVault(URI vaultUri, String vaultToken) { + request = new RestRequest(vaultUri, () -> Arrays.asList( + "X-Vault-Token", vaultToken + )); + } + + String getCookie() { + if ((cookie == null) || Instant.now().isAfter(expires)) { + var result = request.get("").execute(); + cookie = result.get("data").get("cookie.name").asString() + "=" + result.get("data").get("cookie.value").asString(); + expires = Instant.now().plus(Duration.ofSeconds(result.get("lease_duration").asInt()).dividedBy(2)); + log.info("Renewed Jira token (" + cookie + ") - expires " + expires); + } + return cookie; + } +} diff --git a/issuetracker/src/test/java/org/openjdk/skara/issuetracker/IssueTrackerTests.java b/issuetracker/src/test/java/org/openjdk/skara/issuetracker/IssueTrackerTests.java index f53df2d83..a206d9942 100644 --- a/issuetracker/src/test/java/org/openjdk/skara/issuetracker/IssueTrackerTests.java +++ b/issuetracker/src/test/java/org/openjdk/skara/issuetracker/IssueTrackerTests.java @@ -27,8 +27,9 @@ import org.junit.jupiter.api.*; import java.io.IOException; +import java.util.List; -import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.*; class IssueTrackerTests { @Test @@ -39,4 +40,32 @@ void isMemberOfNegativeTests(TestInfo info) throws IOException { assertFalse(host.isMemberOf(madeUpGroupIdThatCannotContainTestMember, host.currentUser())); } } + + @Test + void simple(TestInfo info) throws IOException { + try (var credentials = new HostCredentials(info)) { + var project = credentials.getIssueProject(); + + var userName = project.issueTracker().currentUser().userName(); + var user = project.issueTracker().user(userName); + assertEquals(userName, user.userName()); + + var issue = credentials.createIssue(project, "Test issue"); + issue.setTitle("Updated title"); + issue.setBody("This is now the body"); + var comment = issue.addComment("This is a comment"); + issue.updateComment(comment.id(), "Now it is updated"); + issue.addLabel("label"); + issue.addLabel("another"); + issue.removeLabel("label"); + issue.setAssignees(List.of(project.issueTracker().currentUser())); + + var updated = project.issue(issue.id()).orElseThrow(); + assertEquals(List.of("another"), updated.labels()); + assertEquals(List.of(project.issueTracker().currentUser()), updated.assignees()); + assertEquals(1, updated.comments().size()); + assertEquals("Updated title", updated.title()); + assertEquals("Now it is updated", updated.comments().get(0).body()); + } + } } diff --git a/test/src/main/java/org/openjdk/skara/test/HostCredentials.java b/test/src/main/java/org/openjdk/skara/test/HostCredentials.java index 458b2433c..9ab0d76d7 100644 --- a/test/src/main/java/org/openjdk/skara/test/HostCredentials.java +++ b/test/src/main/java/org/openjdk/skara/test/HostCredentials.java @@ -25,8 +25,8 @@ import org.openjdk.skara.forge.*; import org.openjdk.skara.host.*; import org.openjdk.skara.issuetracker.*; -import org.openjdk.skara.network.URIBuilder; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.URIBuilder; import org.openjdk.skara.proxy.HttpProxy; import org.openjdk.skara.vcs.*; @@ -44,6 +44,7 @@ public class HostCredentials implements AutoCloseable { private final String testName; private final Credentials credentials; private final List pullRequestsToBeClosed = new ArrayList<>(); + private final List issuesToBeClosed = new ArrayList<>(); private HostedRepository credentialsLock; private int nextHostIndex; @@ -141,6 +142,43 @@ public String getNamespaceName() { } } + private static class JiraCredentials implements Credentials { + private final JSONObject config; + + JiraCredentials(JSONObject config) { + this.config = config; + } + + @Override + public Forge createRepositoryHost(int userIndex) { + throw new RuntimeException("not supported"); + } + + @Override + public IssueTracker createIssueHost(int userIndex) { + var hostUri = URIBuilder.base(config.get("host").asString()).build(); + var users = config.get("users").asArray(); + var pat = new Credential(users.get(userIndex).get("name").asString(), + users.get(userIndex).get("pat").asString()); + return IssueTracker.from("jira", hostUri, pat, null); + } + + @Override + public HostedRepository getHostedRepository(Forge host) { + return host.repository(config.get("project").asString()); + } + + @Override + public IssueProject getIssueProject(IssueTracker host) { + return host.project(config.get("project").asString()); + } + + @Override + public String getNamespaceName() { + return config.get("namespace").asString(); + } + } + private static class TestCredentials implements Credentials { private final List hosts = new ArrayList<>(); private final List users = List.of( @@ -200,6 +238,8 @@ private Credentials parseEntry(JSONObject entry, Path credentialsPath) { return new GitLabCredentials(entry); case "github": return new GitHubCredentials(entry, credentialsPath); + case "jira": + return new JiraCredentials(entry); default: throw new RuntimeException("Unknown entry type: " + entry.get("type").asString()); } @@ -322,6 +362,12 @@ public PullRequest createPullRequest(HostedRepository hostedRepository, String t return createPullRequest(hostedRepository, targetRef, sourceRef, title, false); } + public Issue createIssue(IssueProject issueProject, String title) { + var issue = issueProject.createIssue(title, List.of()); + issuesToBeClosed.add(issue); + return issue; + } + public CensusBuilder getCensusBuilder() { return CensusBuilder.create(credentials.getNamespaceName()); } @@ -331,6 +377,9 @@ public void close() { for (var pr : pullRequestsToBeClosed) { pr.setState(PullRequest.State.CLOSED); } + for (var issue : issuesToBeClosed) { + issue.setState(Issue.State.CLOSED); + } if (credentialsLock != null) { try { releaseLock(credentialsLock); diff --git a/test/src/main/java/org/openjdk/skara/test/IssueData.java b/test/src/main/java/org/openjdk/skara/test/IssueData.java index 02ea84e82..860753edd 100644 --- a/test/src/main/java/org/openjdk/skara/test/IssueData.java +++ b/test/src/main/java/org/openjdk/skara/test/IssueData.java @@ -22,6 +22,7 @@ */ package org.openjdk.skara.test; +import org.openjdk.skara.host.HostUser; import org.openjdk.skara.issuetracker.*; import java.time.ZonedDateTime; @@ -33,6 +34,7 @@ class IssueData { String title = ""; final List comments = new ArrayList<>(); final Set labels = new HashSet<>(); + final List assignees = new ArrayList<>(); ZonedDateTime created = ZonedDateTime.now(); ZonedDateTime lastUpdate = created; } diff --git a/test/src/main/java/org/openjdk/skara/test/TestIssue.java b/test/src/main/java/org/openjdk/skara/test/TestIssue.java index fc9d758b8..5ec0f3e33 100644 --- a/test/src/main/java/org/openjdk/skara/test/TestIssue.java +++ b/test/src/main/java/org/openjdk/skara/test/TestIssue.java @@ -170,11 +170,13 @@ public URI webUrl() { @Override public List assignees() { - throw new RuntimeException("not implemented yet"); + return new ArrayList<>(data.assignees); } @Override public void setAssignees(List assignees) { - throw new RuntimeException("not implemented yet"); + data.assignees.clear(); + data.assignees.addAll(assignees); + data.lastUpdate = ZonedDateTime.now(); } } From 0d73a1f81f2f6019eeede509fc951925785d6070 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Tue, 5 Nov 2019 10:41:58 +0000 Subject: [PATCH 17/54] 147: Update the "Changes required" message Reviewed-by: ehelin --- .../java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java | 2 +- .../java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java | 2 +- .../skara/bots/mlbridge/MailingListBridgeBotTests.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java index 5bf2cc949..d85f74b11 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java @@ -147,7 +147,7 @@ static String reviewCommentBody(String body, Review.Verdict verdict, String user if (verdict == Review.Verdict.APPROVED) { result.append("Approved"); } else { - result.append("Disapproved"); + result.append("Changes requested"); } result.append(" by "); result.append(user); diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java index 0e83a5338..a1047eba6 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java @@ -341,7 +341,7 @@ void addReview(Review review) { } var userName = contributor != null ? contributor.username() : review.reviewer().userName() + "@" + censusInstance.namespace().name(); - var userRole = contributor != null ? projectRole(contributor) : "none"; + var userRole = contributor != null ? projectRole(contributor) : "no project role"; var replyBody = ArchiveMessages.reviewCommentBody(review.body().orElse(""), review.verdict(), userName, userRole); addReplyCommon(parent, review.reviewer(), subject, replyBody, id); diff --git a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java index a876f548b..6455cf7d9 100644 --- a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java +++ b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java @@ -1063,7 +1063,7 @@ void notifyReviewVerdicts(TestInfo testInfo) throws IOException { // The archive should contain a note Repository.materialize(archiveFolder.path(), archive.url(), "master"); - assertEquals(1, archiveContainsCount(archiveFolder.path(), "Disapproved by ")); + assertEquals(1, archiveContainsCount(archiveFolder.path(), "Changes requested by ")); assertEquals(1, archiveContainsCount(archiveFolder.path(), " by integrationreviewer1")); if (author.forge().supportsReviewBody()) { assertEquals(1, archiveContainsCount(archiveFolder.path(), "Reason 1")); @@ -1091,7 +1091,7 @@ void notifyReviewVerdicts(TestInfo testInfo) throws IOException { // The archive should contain another note Repository.materialize(archiveFolder.path(), archive.url(), "master"); - assertEquals(2, archiveContainsCount(archiveFolder.path(), "Disapproved by ")); + assertEquals(2, archiveContainsCount(archiveFolder.path(), "Changes requested by ")); if (author.forge().supportsReviewBody()) { assertEquals(1, archiveContainsCount(archiveFolder.path(), "Reason 3")); } From 43f8ff2bcef019e8314e9a3862f2f0357a63908c Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Wed, 6 Nov 2019 11:09:07 +0000 Subject: [PATCH 18/54] Add "Required" title to jcheck-check only if it fails Reviewed-by: rwestberg --- bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java index 675d04ad8..7ce7e3e4e 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java @@ -171,6 +171,7 @@ private void updateCheckBuilder(CheckBuilder checkBuilder, PullRequestCheckIssue if (visitor.isReadyForReview() && additionalErrors.isEmpty()) { checkBuilder.complete(true); } else { + checkBuilder.title("Required"); var summary = Stream.concat(visitor.getMessages().stream(), additionalErrors.stream()) .sorted() .map(m -> "- " + m) @@ -448,7 +449,6 @@ private void updateMergeReadyComment(boolean isReady, String commitMessage, List private void checkStatus() { var checkBuilder = CheckBuilder.create("jcheck", pr.headHash()); - checkBuilder.title("Required"); var censusDomain = censusInstance.configuration().census().domain(); Exception checkException = null; From 12ced194129c37e2a9343b0c96f52fa369ae193a Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Wed, 6 Nov 2019 11:14:47 +0000 Subject: [PATCH 19/54] 149: Improve formatting of bridged emails Reviewed-by: ehelin --- .../bots/mlbridge/CommentPosterWorkItem.java | 5 ++- .../MailingListArchiveReaderBotTests.java | 7 ++-- .../mlbridge/MailingListBridgeBotTests.java | 10 +++--- .../skara/bots/notify/UpdaterTests.java | 21 +++++------ .../org/openjdk/skara/mailinglist/Mbox.java | 15 +++++--- .../mailinglist/mailman/MailmanList.java | 2 +- .../skara/mailinglist/MailmanTests.java | 36 ++++++++++++++----- 7 files changed, 64 insertions(+), 32 deletions(-) diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/CommentPosterWorkItem.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/CommentPosterWorkItem.java index f44accdb1..fe52bbe4a 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/CommentPosterWorkItem.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/CommentPosterWorkItem.java @@ -65,8 +65,11 @@ public boolean concurrentWith(WorkItem other) { private void postNewMessage(Email email) { var marker = String.format(bridgedMailMarker, Base64.getEncoder().encodeToString(email.id().address().getBytes(StandardCharsets.UTF_8))); + var body = marker + "\n" + - "Mailing list message from " + email.author().toString() + "\n\n" + + "*Mailing list message from [" + email.author().fullName().orElse(email.author().localPart()) + + "](mailto:" + email.author().address() + ") on [" + email.sender().localPart() + + "](mailto:" + email.sender().address() + "):*\n\n" + email.body(); pr.addComment(body); } diff --git a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListArchiveReaderBotTests.java b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListArchiveReaderBotTests.java index 445bb9949..d36ef55f5 100644 --- a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListArchiveReaderBotTests.java +++ b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListArchiveReaderBotTests.java @@ -35,7 +35,7 @@ import java.time.Duration; import java.util.*; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.*; class MailingListArchiveReaderBotTests { private void addReply(Conversation conversation, MailingList mailingList, PullRequest pr) { @@ -43,7 +43,7 @@ private void addReply(Conversation conversation, MailingList mailingList, PullRe var reply = "Looks good"; var references = first.id().toString(); - var email = Email.create(EmailAddress.from("Commenter", ""), "Re: RFR: " + pr.title(), reply) + var email = Email.create(EmailAddress.from("Commenter", "c@test.test"), "Re: RFR: " + pr.title(), reply) .recipient(first.author()) .id(EmailAddress.from(UUID.randomUUID() + "@id.id")) .header("In-Reply-To", first.id().toString()) @@ -115,6 +115,9 @@ void simpleArchive(TestInfo testInfo) throws IOException { // The bridge should now have processed the reply var updated = pr.comments(); assertEquals(2, updated.size()); + assertTrue(updated.get(1).body().contains("Mailing list message from")); + assertTrue(updated.get(1).body().contains("[Commenter](mailto:c@test.test)")); + assertTrue(updated.get(1).body().contains("[test](mailto:test@" + listAddress.domain() + ")")); } } diff --git a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java index 6455cf7d9..93fbda79c 100644 --- a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java +++ b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java @@ -197,7 +197,7 @@ void simpleArchive(TestInfo testInfo) throws IOException { assertEquals("RFR: 1234: This is a pull request", mail.subject()); assertEquals(pr.author().fullName(), mail.author().fullName().orElseThrow()); assertEquals(noreplyAddress(archive), mail.author().address()); - assertEquals(from, mail.sender()); + assertEquals(listAddress, mail.sender()); assertEquals("val1", mail.headerValue("Extra1")); assertEquals("val2", mail.headerValue("Extra2")); @@ -250,7 +250,7 @@ void simpleArchive(TestInfo testInfo) throws IOException { assertEquals(3, conversations.get(0).allMessages().size()); for (var newMail : conversations.get(0).allMessages()) { assertEquals(noreplyAddress(archive), newMail.author().address()); - assertEquals(from, newMail.sender()); + assertEquals(listAddress, newMail.sender()); } assertTrue(conversations.get(0).allMessages().get(2).body().contains("This is a comment 😄")); } @@ -340,7 +340,7 @@ void reviewComment(TestInfo testInfo) throws IOException { assertEquals(3, conversations.get(0).allMessages().size()); for (var newMail : conversations.get(0).allMessages()) { assertEquals(noreplyAddress(archive), newMail.author().address()); - assertEquals(from, newMail.sender()); + assertEquals(listAddress, newMail.sender()); } } } @@ -813,7 +813,7 @@ void incrementalChanges(TestInfo testInfo) throws IOException { assertEquals(1, conversations.size()); for (var newMail : conversations.get(0).allMessages()) { assertEquals(noreplyAddress(archive), newMail.author().address()); - assertEquals(from, newMail.sender()); + assertEquals(listAddress, newMail.sender()); } // Add a comment @@ -933,7 +933,7 @@ void rebased(TestInfo testInfo) throws IOException { assertEquals(1, conversations.size()); for (var newMail : conversations.get(0).allMessages()) { assertEquals(noreplyAddress(archive), newMail.author().address()); - assertEquals(sender, newMail.sender()); + assertEquals(listAddress, newMail.sender()); assertFalse(newMail.hasHeader("PR-Head-Hash")); } assertEquals("Re: [Rev 01] RFR: This is a pull request", conversations.get(0).allMessages().get(1).subject()); diff --git a/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java b/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java index d759d5d11..868b753e9 100644 --- a/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java +++ b/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java @@ -200,7 +200,7 @@ void testMailingList(TestInfo testInfo) throws IOException { var conversations = mailmanList.conversations(Duration.ofDays(1)); var email = conversations.get(0).first(); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(sender, email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertTrue(email.subject().contains(": 23456789: More fixes")); @@ -256,7 +256,7 @@ void testMailingListMultiple(TestInfo testInfo) throws IOException { var conversations = mailmanList.conversations(Duration.ofDays(1)); var email = conversations.get(0).first(); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(EmailAddress.from("another_author", "another@author.example.com"), email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertTrue(email.subject().contains(": 2 new changesets")); @@ -306,7 +306,7 @@ void testMailingListSponsored(TestInfo testInfo) throws IOException { var conversations = mailmanList.conversations(Duration.ofDays(1)); var email = conversations.get(0).first(); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(EmailAddress.from("committer", "committer@test.test"), email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertTrue(email.body().contains("Changeset: " + editHash.abbreviate())); @@ -357,7 +357,7 @@ void testMailingListMultipleBranches(TestInfo testInfo) throws IOException { var conversations = mailmanList.conversations(Duration.ofDays(1)); var email = conversations.get(0).first(); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(author, email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertFalse(email.subject().contains("another")); @@ -379,7 +379,8 @@ void testMailingListMultipleBranches(TestInfo testInfo) throws IOException { conversations = mailmanList.conversations(Duration.ofDays(1)); conversations.sort(Comparator.comparing(conversation -> conversation.first().subject())); email = conversations.get(0).first(); - assertEquals(email.sender(), sender); + assertEquals(author, email.author()); + assertEquals(listAddress, email.sender()); assertEquals(email.recipients(), List.of(listAddress)); assertTrue(email.subject().contains(": another: 456789AB: Yet more fixes")); assertFalse(email.subject().contains("master")); @@ -450,7 +451,7 @@ void testMailingListPROnly(TestInfo testInfo) throws IOException { assertEquals(1, conversations.size()); var first = conversations.get(0).first(); var email = conversations.get(0).replies(first).get(0); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(author, email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertEquals("Re: [Integrated] RFR: My PR", email.subject()); @@ -537,7 +538,7 @@ void testMailingListPR(TestInfo testInfo) throws IOException { var pushConversation = conversations.get(1); var prEmail = prConversation.replies(prConversation.first()).get(0); - assertEquals(sender, prEmail.sender()); + assertEquals(listAddress, prEmail.sender()); assertEquals(EmailAddress.from("testauthor", "ta@none.none"), prEmail.author()); assertEquals(prEmail.recipients(), List.of(listAddress)); assertEquals("Re: [Integrated] RFR: My PR", prEmail.subject()); @@ -548,7 +549,7 @@ void testMailingListPR(TestInfo testInfo) throws IOException { assertFalse(prEmail.body().contains(masterHash.abbreviate())); var pushEmail = pushConversation.first(); - assertEquals(sender, pushEmail.sender()); + assertEquals(listAddress, pushEmail.sender()); assertEquals(EmailAddress.from("testauthor", "ta@none.none"), pushEmail.author()); assertEquals(pushEmail.recipients(), List.of(listAddress)); assertTrue(pushEmail.subject().contains("23456789: More fixes")); @@ -686,7 +687,7 @@ void testMailingListBranch(TestInfo testInfo) throws IOException { var conversations = mailmanList.conversations(Duration.ofDays(1)); var email = conversations.get(0).first(); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(EmailAddress.from("testauthor", "ta@none.none"), email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertEquals("git: test: created branch newbranch1 based on the branch master containing 2 unique commits", email.subject()); @@ -707,7 +708,7 @@ void testMailingListBranch(TestInfo testInfo) throws IOException { .filter(c -> !c.equals(conversations.get(0))) .findFirst().orElseThrow(); email = newConversation.first(); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(sender, email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertEquals("git: test: created branch newbranch2 based on the branch newbranch1 containing 0 unique commits", email.subject()); diff --git a/mailinglist/src/main/java/org/openjdk/skara/mailinglist/Mbox.java b/mailinglist/src/main/java/org/openjdk/skara/mailinglist/Mbox.java index ad9ac60c1..c680f5d55 100644 --- a/mailinglist/src/main/java/org/openjdk/skara/mailinglist/Mbox.java +++ b/mailinglist/src/main/java/org/openjdk/skara/mailinglist/Mbox.java @@ -42,7 +42,7 @@ public class Mbox { private final static Pattern fromStringEncodePattern = Pattern.compile("^(>*From )", Pattern.MULTILINE); private final static Pattern fromStringDecodePattern = Pattern.compile("^>(>*From )", Pattern.MULTILINE); - private static List splitMbox(String mbox) { + private static List splitMbox(String mbox, EmailAddress sender) { // Initial split var messages = mboxMessagePattern.matcher(mbox).results() .map(match -> match.group(1)) @@ -57,8 +57,11 @@ private static List splitMbox(String mbox) { for (var message : messages) { messageBuilder.insert(0, message); try { - var email = Email.parse(messageBuilder.toString()); - parsedMails.add(email); + var email = Email.from(Email.parse(messageBuilder.toString())); + if (sender != null) { + email.sender(sender); + } + parsedMails.add(email.build()); messageBuilder.setLength(0); } catch (RuntimeException ignored) { } @@ -79,7 +82,11 @@ private static String decodeFromStrings(String body) { } public static List parseMbox(String mbox) { - var emails = splitMbox(mbox); + return parseMbox(mbox, null); + } + + public static List parseMbox(String mbox, EmailAddress sender) { + var emails = splitMbox(mbox, sender); var idToMail = emails.stream().collect(Collectors.toMap(Email::id, Function.identity(), (a, b) -> a)); var idToConversation = idToMail.values().stream() .filter(email -> !email.hasHeader("In-Reply-To")) diff --git a/mailinglist/src/main/java/org/openjdk/skara/mailinglist/mailman/MailmanList.java b/mailinglist/src/main/java/org/openjdk/skara/mailinglist/mailman/MailmanList.java index c8cae0860..b2f4f1fd6 100644 --- a/mailinglist/src/main/java/org/openjdk/skara/mailinglist/mailman/MailmanList.java +++ b/mailinglist/src/main/java/org/openjdk/skara/mailinglist/mailman/MailmanList.java @@ -143,7 +143,7 @@ public List conversations(Duration maxAge) { if (newContent) { var concatenatedMbox = String.join("", actualPages); - var mails = Mbox.parseMbox(concatenatedMbox); + var mails = Mbox.parseMbox(concatenatedMbox, listAddress); var threshold = ZonedDateTime.now().minus(maxAge); cachedConversations = mails.stream() .filter(mail -> mail.first().date().isAfter(threshold)) diff --git a/mailinglist/src/test/java/org/openjdk/skara/mailinglist/MailmanTests.java b/mailinglist/src/test/java/org/openjdk/skara/mailinglist/MailmanTests.java index 623bf257e..f802673f9 100644 --- a/mailinglist/src/test/java/org/openjdk/skara/mailinglist/MailmanTests.java +++ b/mailinglist/src/test/java/org/openjdk/skara/mailinglist/MailmanTests.java @@ -45,12 +45,16 @@ void simple() throws IOException { .recipient(EmailAddress.parse(listAddress)) .build(); mailmanList.post(mail); + var expectedMail = Email.from(mail) + .sender(EmailAddress.parse(listAddress)) + .build(); + testServer.processIncoming(); var conversations = mailmanList.conversations(Duration.ofDays(1)); assertEquals(1, conversations.size()); var conversation = conversations.get(0); - assertEquals(mail, conversation.first()); + assertEquals(expectedMail, conversation.first()); } } @@ -67,11 +71,14 @@ void replies() throws IOException { .build(); mailmanList.post(sentParent); testServer.processIncoming(); + var expectedParent = Email.from(sentParent) + .sender(EmailAddress.parse(listAddress)) + .build(); var conversations = mailmanList.conversations(Duration.ofDays(1)); assertEquals(1, conversations.size()); var conversation = conversations.get(0); - assertEquals(sentParent, conversation.first()); + assertEquals(expectedParent, conversation.first()); var replier = EmailAddress.from("Replier", "replier@test.email"); var sentReply = Email.create(replier, "Reply subject", "Reply body") @@ -79,17 +86,21 @@ void replies() throws IOException { .header("In-Reply-To", sentParent.id().toString()) .build(); mailmanList.post(sentReply); + var expectedReply = Email.from(sentReply) + .sender(EmailAddress.parse(listAddress)) + .build(); + testServer.processIncoming(); conversations = mailmanList.conversations(Duration.ofDays(1)); assertEquals(1, conversations.size()); conversation = conversations.get(0); - assertEquals(sentParent, conversation.first()); + assertEquals(expectedParent, conversation.first()); var replies = conversation.replies(conversation.first()); assertEquals(1, replies.size()); var reply = replies.get(0); - assertEquals(sentReply, reply); + assertEquals(expectedReply, reply); } } @@ -107,18 +118,21 @@ void cached() throws IOException { mailmanList.post(mail); testServer.processIncoming(); + var expectedMail = Email.from(mail) + .sender(EmailAddress.parse(listAddress)) + .build(); { var conversations = mailmanList.conversations(Duration.ofDays(1)); assertEquals(1, conversations.size()); var conversation = conversations.get(0); - assertEquals(mail, conversation.first()); + assertEquals(expectedMail, conversation.first()); assertFalse(testServer.lastResponseCached()); } { var conversations = mailmanList.conversations(Duration.ofDays(1)); assertEquals(1, conversations.size()); var conversation = conversations.get(0); - assertEquals(mail, conversation.first()); + assertEquals(expectedMail, conversation.first()); assertTrue(testServer.lastResponseCached()); } } @@ -133,8 +147,8 @@ void interval() throws IOException { var mailmanList = mailmanServer.getList(listAddress); var sender = EmailAddress.from("Test", "test@test.email"); var mail1 = Email.create(sender, "Subject 1", "Body 1") - .recipient(EmailAddress.parse(listAddress)) - .build(); + .recipient(EmailAddress.parse(listAddress)) + .build(); var mail2 = Email.create(sender, "Subject 2", "Body 2") .recipient(EmailAddress.parse(listAddress)) .build(); @@ -142,13 +156,17 @@ void interval() throws IOException { mailmanList.post(mail1); mailmanList.post(mail2); }).start(); + var expectedMail = Email.from(mail1) + .sender(EmailAddress.parse(listAddress)) + .build(); + testServer.processIncoming(); assertThrows(RuntimeException.class, () -> testServer.processIncoming(Duration.ZERO)); var conversations = mailmanList.conversations(Duration.ofDays(1)); assertEquals(1, conversations.size()); var conversation = conversations.get(0); - assertEquals(mail1, conversation.first()); + assertEquals(expectedMail, conversation.first()); } } } From f4cc08e26bb533f83f3336e0d174f7669c3fcdc4 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Wed, 6 Nov 2019 16:36:26 +0000 Subject: [PATCH 20/54] 148: Review comment general comment first Reviewed-by: ehelin --- .../skara/bots/mlbridge/ArchiveMessages.java | 11 ++++-- .../skara/bots/mlbridge/ArchiveWorkItem.java | 14 +++++-- .../skara/bots/mlbridge/ReviewArchive.java | 37 +++++++++++++++++-- .../mlbridge/MailingListBridgeBotTests.java | 12 ++++-- .../java/org/openjdk/skara/email/Email.java | 8 ++++ .../org/openjdk/skara/email/EmailTests.java | 18 +++++++++ 6 files changed, 86 insertions(+), 14 deletions(-) diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java index d85f74b11..c6e630bf1 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java @@ -136,10 +136,15 @@ static String composeCombinedReply(Email parent, String body, PullRequestInstanc replyFooter(prInstance); } - static String reviewCommentBody(String body, Review.Verdict verdict, String user, String role) { - var result = new StringBuilder(filterComments(body)); + static String reviewCommentBody(String body) { + return filterComments(body); + } + + static String reviewVerdictBody(String body, Review.Verdict verdict, String user, String role) { + var filteredBody = filterComments(body); + var result = new StringBuilder(); if (verdict != Review.Verdict.NONE) { - if (result.length() > 0) { + if (filteredBody.length() > 0) { result.append("\n\n"); result.append(infoSeparator); result.append("\n\n"); diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveWorkItem.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveWorkItem.java index 665ab08d9..b0dbdcd66 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveWorkItem.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveWorkItem.java @@ -281,6 +281,15 @@ public void run(Path scratchPath) { reviewArchive.addComment(comment); } + // Review comments + var reviews = pr.reviews(); + for (var review : reviews) { + if (ignoreComment(review.reviewer(), review.body().orElse(""))) { + continue; + } + reviewArchive.addReview(review); + } + // File specific comments var reviewComments = pr.reviewComments(); for (var reviewComment : reviewComments) { @@ -290,13 +299,12 @@ public void run(Path scratchPath) { reviewArchive.addReviewComment(reviewComment); } - // Review comments - var reviews = pr.reviews(); + // Review verdict comments for (var review : reviews) { if (ignoreComment(review.reviewer(), review.body().orElse(""))) { continue; } - reviewArchive.addReview(review); + reviewArchive.addReviewVerdict(review); } var newMails = reviewArchive.generatedEmails(); diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java index a1047eba6..ac1ce5d08 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java @@ -21,6 +21,7 @@ class ReviewArchive { private final Map existingIds = new HashMap<>(); private final List generated = new ArrayList<>(); private final Map generatedIds = new HashMap<>(); + private final Set approvalIds = new HashSet<>(); private final List reportedHeads; private final List reportedBases; @@ -327,6 +328,21 @@ void addReview(Review review) { return; } + // Default parent and subject + var parent = topCommentForHash(review.hash()); + var subject = parent.subject(); + + var replyBody = ArchiveMessages.reviewCommentBody(review.body().orElse("")); + + addReplyCommon(parent, review.reviewer(), subject, replyBody, id); + } + + void addReviewVerdict(Review review) { + var id = getMessageId(review); + if (existingIds.containsKey(getStableMessageId(id))) { + return; + } + var contributor = censusInstance.namespace().get(review.reviewer().id()); var isReviewer = contributor != null && censusInstance.project().isReviewer(contributor.username(), censusInstance.configuration().census().version()); @@ -336,13 +352,12 @@ void addReview(Review review) { // Approvals by Reviewers get special treatment - post these as top-level comments if (review.verdict() == Review.Verdict.APPROVED && isReviewer) { - parent = topEmail(); - subject = "Re: [Approved] " + "RFR: " + prInstance.pr().title(); + approvalIds.add(id); } var userName = contributor != null ? contributor.username() : review.reviewer().userName() + "@" + censusInstance.namespace().name(); var userRole = contributor != null ? projectRole(contributor) : "no project role"; - var replyBody = ArchiveMessages.reviewCommentBody(review.body().orElse(""), review.verdict(), userName, userRole); + var replyBody = ArchiveMessages.reviewVerdictBody(review.body().orElse(""), review.verdict(), userName, userRole); addReplyCommon(parent, review.reviewer(), subject, replyBody, id); } @@ -372,6 +387,20 @@ void addReviewComment(ReviewComment reviewComment) { } List generatedEmails() { - return generated; + var finalEmails = new ArrayList(); + for (var email : generated) { + for (var approvalId : approvalIds) { + var collapsed = email.hasHeader("PR-Collapsed-IDs") ? email.headerValue("PR-Collapsed-IDs") + " " : ""; + if (email.id().equals(approvalId) || collapsed.contains(getStableMessageId(approvalId))) { + email = Email.reparent(topEmail(), email) + .subject("Re: [Approved] " + "RFR: " + prInstance.pr().title()) + .build(); + break; + } + } + finalEmails.add(email); + } + + return finalEmails; } } diff --git a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java index 93fbda79c..d201e5b0c 100644 --- a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java +++ b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java @@ -491,6 +491,7 @@ void commentThreading(TestInfo testInfo) throws IOException { // Finally some approvals and another comment pr.addReview(Review.Verdict.APPROVED, "Nice"); + reviewPr.addReviewComment(masterHash, editHash, reviewFile.toString(), 2, "The final review comment"); reviewPr.addReview(Review.Verdict.APPROVED, "Looks fine"); reviewPr.addReviewCommentReply(comment2, "You are welcome"); TestBotRunner.runPeriodicItems(mlBot); @@ -502,9 +503,9 @@ void commentThreading(TestInfo testInfo) throws IOException { Repository.materialize(archiveFolder.path(), archive.url(), "master"); assertEquals(9, archiveContainsCount(archiveFolder.path(), "^On.*wrote:")); - // File specific comments should appear before the approval + // File specific comments should appear after the approval var archiveText = archiveContents(archiveFolder.path()).orElseThrow(); - assertTrue(archiveText.indexOf("Looks fine") > archiveText.indexOf("You are welcome")); + assertTrue(archiveText.indexOf("Looks fine") < archiveText.indexOf("The final review comment")); // Check the mailing list var mailmanServer = MailingListServerFactory.createMailmanServer(listServer.getArchive(), listServer.getSMTP(), Duration.ZERO); @@ -543,10 +544,13 @@ void commentThreading(TestInfo testInfo) throws IOException { assertTrue(thread2reply2.body().contains("Thanks")); var replies = conversations.get(0).replies(mail); - var thread3 = conversations.get(0).replies(mail).get(2); + var thread3 = replies.get(2); assertEquals("Re: RFR: This is a pull request", thread3.subject()); - var thread4 = conversations.get(0).replies(mail).get(3); + var thread4 = replies.get(3); assertEquals("Re: [Approved] RFR: This is a pull request", thread4.subject()); + assertTrue(thread4.body().contains("Looks fine")); + assertTrue(thread4.body().contains("The final review comment")); + assertTrue(thread4.body().contains("Approved by integrationreviewer1 (Reviewer)")); } } diff --git a/email/src/main/java/org/openjdk/skara/email/Email.java b/email/src/main/java/org/openjdk/skara/email/Email.java index 8d06b596f..aafe09318 100644 --- a/email/src/main/java/org/openjdk/skara/email/Email.java +++ b/email/src/main/java/org/openjdk/skara/email/Email.java @@ -146,6 +146,14 @@ public static EmailBuilder reply(Email parent, String subject, String body) { .header("References", references); } + public static EmailBuilder reparent(Email newParent, Email email) { + var currentParent = email.headerValue("In-Reply-To"); + var currentRefs = email.headerValue("References"); + + return from(email).header("In-Reply-To", newParent.id.toString()) + .header("References", currentRefs.replace(currentParent, newParent.id.toString())); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/email/src/test/java/org/openjdk/skara/email/EmailTests.java b/email/src/test/java/org/openjdk/skara/email/EmailTests.java index 1bc0ac4f3..9fb38726e 100644 --- a/email/src/test/java/org/openjdk/skara/email/EmailTests.java +++ b/email/src/test/java/org/openjdk/skara/email/EmailTests.java @@ -67,6 +67,24 @@ void buildFrom() { assertEquals(original, copy); } + @Test + void reparent() { + var first = Email.create(EmailAddress.from("A", "a@b.c"), "First", "body") + .recipient(EmailAddress.from("B", "b@b.c")) + .build(); + var second = Email.create(EmailAddress.from("A", "a@b.c"), "Second", "body") + .recipient(EmailAddress.from("B", "b@b.c")) + .build(); + var reply = Email.reply(first, "The reply", "reply body") + .author(EmailAddress.from("C", "c@b.c")) + .build(); + assertEquals(first.id().toString(), reply.headerValue("In-Reply-To")); + assertEquals(first.id().toString(), reply.headerValue("References")); + var updated = Email.reparent(second, reply).build(); + assertEquals(second.id().toString(), updated.headerValue("In-Reply-To")); + assertEquals(second.id().toString(), updated.headerValue("References")); + } + @Test void caseInsensitiveHeaders() { var mail = Email.parse("Message-ID: \n" + From b1e3d870862f9ef0b7adcb015fd5c3ae243c25ec Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Thu, 7 Nov 2019 10:59:57 +0000 Subject: [PATCH 21/54] 151: Encode single dot characters in the SMTP client Reviewed-by: ehelin --- .../java/org/openjdk/skara/email/SMTP.java | 12 ++++++---- .../org/openjdk/skara/email/SMTPTests.java | 23 +++++++++++-------- .../org/openjdk/skara/test/SMTPServer.java | 3 +++ 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/email/src/main/java/org/openjdk/skara/email/SMTP.java b/email/src/main/java/org/openjdk/skara/email/SMTP.java index 0884db4a0..28b655ac6 100644 --- a/email/src/main/java/org/openjdk/skara/email/SMTP.java +++ b/email/src/main/java/org/openjdk/skara/email/SMTP.java @@ -28,18 +28,19 @@ import java.time.Duration; import java.time.format.DateTimeFormatter; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Limited SMTP client implementation - only compatibility requirement (currently) is the OpenJDK * mailing list servers. */ public class SMTP { - private static Pattern initReply = Pattern.compile("220 .*"); + private static Pattern initReply = Pattern.compile("^220 .*"); private static Pattern ehloReply = Pattern.compile("^250 .*"); private static Pattern mailReply = Pattern.compile("^250 .*"); private static Pattern rcptReply = Pattern.compile("^250 .*"); - private static Pattern dataReply = Pattern.compile("354 Enter.*"); - private static Pattern doneReply = Pattern.compile("250 .*"); + private static Pattern dataReply = Pattern.compile("^354 .*"); + private static Pattern doneReply = Pattern.compile("^250 .*"); public static void send(String server, EmailAddress recipient, Email email) throws IOException { send(server, recipient, email, Duration.ofMinutes(30)); @@ -74,7 +75,10 @@ public static void send(String server, EmailAddress recipient, Email email, Dura session.sendCommand("Subject: " + MimeText.encode(email.subject())); session.sendCommand("Content-type: text/plain; charset=utf-8"); session.sendCommand(""); - session.sendCommand(email.body()); + var escapedBody = email.body().lines() + .map(line -> line.startsWith(".") ? "." + line : line) + .collect(Collectors.joining("\n")); + session.sendCommand(escapedBody); session.sendCommand(".", doneReply); session.sendCommand("QUIT"); } diff --git a/email/src/test/java/org/openjdk/skara/email/SMTPTests.java b/email/src/test/java/org/openjdk/skara/email/SMTPTests.java index 3fc5ea7c4..0cb7ab506 100644 --- a/email/src/test/java/org/openjdk/skara/email/SMTPTests.java +++ b/email/src/test/java/org/openjdk/skara/email/SMTPTests.java @@ -22,24 +22,19 @@ */ package org.openjdk.skara.email; -import org.junit.jupiter.api.condition.DisabledOnOs; -import org.junit.jupiter.api.condition.OS; import org.openjdk.skara.test.SMTPServer; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.*; import java.io.IOException; import java.time.Duration; -import java.util.logging.Logger; import static org.junit.jupiter.api.Assertions.*; class SMTPTests { - private final static Logger log = Logger.getLogger("org.openjdk.skara.email");; - @Test void simple() throws IOException { - log.info("Hello"); try (var server = new SMTPServer()) { var sender = EmailAddress.from("Test", "test@test.email"); var recipient = EmailAddress.from("Dest", "dest@dest.email"); @@ -53,7 +48,6 @@ void simple() throws IOException { @Test void withHeader() throws IOException { - log.info("Hello"); try (var server = new SMTPServer()) { var sender = EmailAddress.from("Test", "test@test.email"); var author = EmailAddress.from("Auth", "auth@test.email"); @@ -73,7 +67,6 @@ void withHeader() throws IOException { @Test @DisabledOnOs(OS.WINDOWS) void encoded() throws IOException { - log.info("Hello"); try (var server = new SMTPServer()) { var sender = EmailAddress.from("Señor Dévèlöper", "test@test.email"); var recipient = EmailAddress.from("Dêst", "dest@dest.email"); @@ -90,7 +83,6 @@ void encoded() throws IOException { @Test void timeout() throws IOException { - log.info("Hello"); try (var server = new SMTPServer()) { var sender = EmailAddress.from("Test", "test@test.email"); var recipient = EmailAddress.from("Dest", "dest@dest.email"); @@ -99,4 +91,17 @@ void timeout() throws IOException { assertThrows(RuntimeException.class, () -> SMTP.send(server.address(), recipient, sentMail, Duration.ZERO)); } } + + @Test + void withDot() throws IOException { + try (var server = new SMTPServer()) { + var sender = EmailAddress.from("Test", "test@test.email"); + var recipient = EmailAddress.from("Dest", "dest@dest.email"); + var sentMail = Email.create(sender, "Subject", "Body\n.\nMore text").recipient(recipient).build(); + + SMTP.send(server.address(), recipient, sentMail); + var email = server.receive(Duration.ofSeconds(10)); + assertEquals(sentMail, email); + } + } } diff --git a/test/src/main/java/org/openjdk/skara/test/SMTPServer.java b/test/src/main/java/org/openjdk/skara/test/SMTPServer.java index bc41e59b6..eed3a74fd 100644 --- a/test/src/main/java/org/openjdk/skara/test/SMTPServer.java +++ b/test/src/main/java/org/openjdk/skara/test/SMTPServer.java @@ -71,6 +71,9 @@ private void handleSession(SMTPSession session) throws IOException { inHeader = false; } } + if (line.startsWith(".")) { + line = line.substring(1); + } mailBody.append(line); mailBody.append("\n"); } From e294ce92b738663b8b991eee71cbfa076ccad96b Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Thu, 7 Nov 2019 13:08:46 +0000 Subject: [PATCH 22/54] Add ReadOnlyRepository.annotate(Tag t) method Reviewed-by: rwestberg --- .../openjdk/skara/jcheck/TestRepository.java | 4 ++ .../openjdk/skara/vcs/ReadOnlyRepository.java | 2 + .../main/java/org/openjdk/skara/vcs/Tag.java | 65 +++++++++++++++++++ .../openjdk/skara/vcs/git/GitRepository.java | 21 ++++++ .../openjdk/skara/vcs/hg/HgRepository.java | 25 +++++++ .../openjdk/skara/vcs/RepositoryTests.java | 50 ++++++++++++++ 6 files changed, 167 insertions(+) diff --git a/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java b/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java index 0ee1af6aa..4e1d2ce29 100644 --- a/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java +++ b/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java @@ -239,4 +239,8 @@ public void addSubmodule(String pullPath, Path path) throws IOException { public List submodules() throws IOException { return null; } + + public Optional annotate(Tag tag) throws IOException { + return null; + } } diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/ReadOnlyRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/ReadOnlyRepository.java index 3490f7673..88e7ee9da 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/ReadOnlyRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/ReadOnlyRepository.java @@ -95,4 +95,6 @@ static Optional get(Path p) throws IOException { static boolean exists(Path p) throws IOException { return Repository.exists(p); } + + Optional annotate(Tag tag) throws IOException; } diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/Tag.java b/vcs/src/main/java/org/openjdk/skara/vcs/Tag.java index 61b4ad785..7c8d16ce3 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/Tag.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/Tag.java @@ -24,8 +24,73 @@ import java.util.Objects; import java.util.Optional; +import java.time.ZonedDateTime; public class Tag { + public static class Annotated { + private final String name; + private final Hash target; + private final Author author; + private final ZonedDateTime date; + private final String message; + + public Annotated(String name, Hash target, Author author, ZonedDateTime date, String message) { + this.name = name; + this.target = target; + this.author = author; + this.date = date; + this.message = message; + } + + public String name() { + return name; + } + + public Hash target() { + return target; + } + + public Author author() { + return author; + } + + public ZonedDateTime date() { + return date; + } + + public String message() { + return message; + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + + if (!(other instanceof Annotated)) { + return false; + } + + var o = (Annotated) other; + return Objects.equals(name, o.name) && + Objects.equals(target, o.target) && + Objects.equals(author, o.author) && + Objects.equals(date, o.date) && + Objects.equals(message, o.message); + } + + @Override + public int hashCode() { + return Objects.hash(name, target, author, date, message); + } + + @Override + public String toString() { + return name + " -> " + target.hex(); + } + } + private final String name; public Tag(String name) { diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java index 49bbb783c..6e2847f6d 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java @@ -1150,4 +1150,25 @@ public List submodules() throws IOException { return modules; } + + @Override + public Optional annotate(Tag tag) throws IOException { + var ref = "refs/tags/" + tag.name(); + var format = "%(refname:short)%0a%(*objectname)%0a%(taggername) %(taggeremail)%0a%(taggerdate:iso-strict)%0a%(contents)"; + try (var p = capture("git", "for-each-ref", "--format", format, ref)) { + var lines = await(p).stdout(); + if (lines.size() >= 4) { + var name = lines.get(0); + var target = new Hash(lines.get(1)); + var author = Author.fromString(lines.get(2)); + + var formatter = DateTimeFormatter.ISO_OFFSET_DATE_TIME; + var date = ZonedDateTime.parse(lines.get(3), formatter); + var message = String.join("\n", lines.subList(4, lines.size())); + + return Optional.of(new Tag.Annotated(name, target, author, date, message)); + } + return Optional.empty(); + } + } } diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java index 1ed12e3b3..59c391a93 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java @@ -1170,4 +1170,29 @@ public List submodules() throws IOException { return modules; } + + @Override + public Optional annotate(Tag tag) throws IOException { + var hgtags = root().resolve(".hgtags"); + if (!Files.exists(hgtags)) { + return Optional.empty(); + } + try (var p = capture("hg", "annotate", hgtags.toString())) { + var reversed = new ArrayList<>(await(p).stdout()); + Collections.reverse(reversed); + for (var line : reversed) { + var parts = line.split(" "); + var tagName = parts[2]; + if (tagName.equals(tag.name())) { + var target = new Hash(parts[1]); + var rev = parts[0].substring(0, parts[0].length() - 1).trim(); // skip last ':' and ev. whitespace + var hash = resolve(rev).orElseThrow(IOException::new); + var commit = lookup(hash).orElseThrow(IOException::new); + var message = String.join("\n", commit.message()) + "\n"; + return Optional.of(new Tag.Annotated(tagName, target, commit.author(), commit.date(), message)); + } + } + } + return Optional.empty(); + } } diff --git a/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java b/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java index 0f875d035..1b1c1edea 100644 --- a/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java +++ b/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java @@ -33,6 +33,7 @@ import java.net.URI; import java.nio.file.*; import java.nio.file.attribute.*; +import java.time.ZonedDateTime; import java.util.*; import java.util.stream.Collectors; @@ -1894,4 +1895,53 @@ void testSubmodulesOnRepoWithSubmodule(VCS vcs) throws IOException { assertEquals(pullPath, module.pullPath()); } } + + @ParameterizedTest + @EnumSource(VCS.class) + void testAnnotateTag(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory()) { + var repo = Repository.init(dir.path(), vcs); + var readme = repo.root().resolve("README"); + var now = ZonedDateTime.now(); + Files.writeString(readme, "Hello\n"); + repo.add(readme); + var head = repo.commit("Added README", "duke", "duke@openjdk.org"); + var tag = repo.tag(head, "1.0", "Added tag 1.0 for HEAD\n", "duke", "duke@openjdk.org"); + var annotated = repo.annotate(tag).get(); + + assertEquals("1.0", annotated.name()); + assertEquals(head, annotated.target()); + assertEquals(new Author("duke", "duke@openjdk.org"), annotated.author()); + assertEquals(now.getYear(), annotated.date().getYear()); + assertEquals(now.getMonth(), annotated.date().getMonth()); + assertEquals(now.getDayOfYear(), annotated.date().getDayOfYear()); + assertEquals(now.getHour(), annotated.date().getHour()); + assertEquals(now.getOffset(), annotated.date().getOffset()); + assertEquals("Added tag 1.0 for HEAD\n", annotated.message()); + } + } + + @ParameterizedTest + @EnumSource(VCS.class) + void testAnnotateTagOnMissingTag(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory()) { + var repo = Repository.init(dir.path(), vcs); + var readme = repo.root().resolve("README"); + var now = ZonedDateTime.now(); + Files.writeString(readme, "Hello\n"); + repo.add(readme); + var head = repo.commit("Added README", "duke", "duke@openjdk.org"); + + assertEquals(Optional.empty(), repo.annotate(new Tag("unknown"))); + } + } + + @ParameterizedTest + @EnumSource(VCS.class) + void testAnnotateTagOnEmptyRepo(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory()) { + var repo = Repository.init(dir.path(), vcs); + assertEquals(Optional.empty(), repo.annotate(new Tag("unknown"))); + } + } } From 069620bcd3a2dc8128431f2b473c63b23eaf647d Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Fri, 8 Nov 2019 08:38:45 +0000 Subject: [PATCH 23/54] Add buildnum extraction support for OpenJFX tags Reviewed-by: kcr, ehelin --- .../openjdk/skara/vcs/openjdk/OpenJDKTag.java | 8 ++++++- .../skara/vcs/openjdk/OpenJDKTagTests.java | 23 +++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/openjdk/OpenJDKTag.java b/vcs/src/main/java/org/openjdk/skara/vcs/openjdk/OpenJDKTag.java index 37df3e33d..6969123dd 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/openjdk/OpenJDKTag.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/openjdk/OpenJDKTag.java @@ -52,16 +52,19 @@ private OpenJDKTag(Tag tag, String prefix, String version, String buildPrefix, S * jdk7u40-b20 -> jdk7u40 7u40 u20 -b 29 * hs24-b30 -> hs24 24 -b 30 * hs23.6-b19 -> hs23.6 23.6 .6 -b 19 + * 11.1+22 -> 11.1 11.1 .1 + 22 */ private final static String legacyOpenJDKVersionPattern = "(jdk([0-9]{1,2}(u[0-9]{1,3})?))"; private final static String legacyHSVersionPattern = "((hs[0-9]{1,2}(\\.[0-9]{1,3})?))"; private final static String legacyBuildPattern = "(-b)([0-9]{2,3})"; private final static String OpenJDKVersionPattern = "(jdk-([0-9]+(\\.[0-9]){0,3}))(\\+)([0-9]+)"; + private final static String OpenJFXVersionPattern = "((?:jdk-){0,1}([1-9](?:(?:[0-9]*)(\\.(?:0|[1-9][0-9]*)){0,3})))(?:(\\+)([0-9]+)|(-ga))"; private final static List tagPatterns = List.of(Pattern.compile(legacyOpenJDKVersionPattern + legacyBuildPattern), Pattern.compile(legacyHSVersionPattern + legacyBuildPattern), - Pattern.compile(OpenJDKVersionPattern)); + Pattern.compile(OpenJDKVersionPattern), + Pattern.compile(OpenJFXVersionPattern)); /** * Attempts to create an OpenJDKTag instance from a general Tag. @@ -106,6 +109,9 @@ public String version() { * @return */ public int buildNum() { + if (buildNum == null) { + return 0; + } return Integer.parseInt(buildNum); } diff --git a/vcs/src/test/java/org/openjdk/skara/vcs/openjdk/OpenJDKTagTests.java b/vcs/src/test/java/org/openjdk/skara/vcs/openjdk/OpenJDKTagTests.java index 70a52dbf1..dd0ef2298 100644 --- a/vcs/src/test/java/org/openjdk/skara/vcs/openjdk/OpenJDKTagTests.java +++ b/vcs/src/test/java/org/openjdk/skara/vcs/openjdk/OpenJDKTagTests.java @@ -22,10 +22,11 @@ */ package org.openjdk.skara.vcs.openjdk; +import org.openjdk.skara.vcs.Tag; + import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; -import org.openjdk.skara.vcs.Tag; +import static org.junit.jupiter.api.Assertions.*; class OpenJDKTagTests { @Test @@ -92,4 +93,22 @@ void noPrevious() { assertEquals(0, jdkTag.buildNum()); assertFalse(jdkTag.previous().isPresent()); } + + @Test + void parseJfxTags() { + var tag = new Tag("12.1.3+14"); + var jdkTag = OpenJDKTag.create(tag).orElseThrow(); + assertEquals("12.1.3", jdkTag.version()); + assertEquals(14, jdkTag.buildNum()); + var previousTag = jdkTag.previous().orElseThrow(); + assertEquals(13, previousTag.buildNum()); + } + + @Test + void parseJfxTagsGa() { + var tag = new Tag("12.1-ga"); + var jdkTag = OpenJDKTag.create(tag).orElseThrow(); + assertEquals("12.1", jdkTag.version()); + assertEquals(0, jdkTag.buildNum()); + } } From 2cb3a2e4ed7324f88fda409fd53198807572349a Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Fri, 8 Nov 2019 08:39:57 +0000 Subject: [PATCH 24/54] Use tag annotation if present when sending notifications Reviewed-by: ehelin --- .../openjdk/skara/bots/notify/JNotifyBot.java | 46 ++++++-- .../skara/bots/notify/JsonUpdater.java | 7 +- .../skara/bots/notify/MailingListUpdater.java | 110 ++++++++++++++---- .../skara/bots/notify/UpdateConsumer.java | 10 +- .../skara/bots/notify/UpdaterTests.java | 17 ++- 5 files changed, 148 insertions(+), 42 deletions(-) diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBot.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBot.java index a2c755b0b..32b236b61 100644 --- a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBot.java +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBot.java @@ -35,7 +35,7 @@ import java.util.*; import java.util.logging.Logger; import java.util.regex.Pattern; -import java.util.stream.Collectors; +import java.util.stream.*; class JNotifyBot implements Bot, WorkItem { private final Logger log = Logger.getLogger("org.openjdk.skara.bots");; @@ -166,33 +166,55 @@ private void handleTags(Repository localRepo, UpdateHistory history) throws IOEx .map(Optional::get) .collect(Collectors.toSet()); var newJdkTags = newTags.stream() - .map(OpenJDKTag::create) - .filter(Optional::isPresent) - .map(Optional::get) - .sorted(Comparator.comparingInt(OpenJDKTag::buildNum)) - .collect(Collectors.toList()); - + .map(OpenJDKTag::create) + .filter(Optional::isPresent) + .map(Optional::get) + .sorted(Comparator.comparingInt(OpenJDKTag::buildNum)) + .collect(Collectors.toList()); for (var tag : newJdkTags) { // Update the history first - if there is a problem here we don't want to send out multiple updates history.addTags(List.of(tag.tag())); var commits = new ArrayList(); + + // Try to determine which commits are new since the last build var previous = existingPrevious(tag, allJdkTags); - if (previous.isEmpty()) { + if (previous.isPresent()) { + commits.addAll(localRepo.commits(previous.get().tag() + ".." + tag.tag()).asList()); + } + + // If none are found, just include the commit that was tagged + if (commits.isEmpty()) { var commit = localRepo.lookup(tag.tag()); if (commit.isEmpty()) { throw new RuntimeException("Failed to lookup tag '" + tag.toString() + "'"); } else { commits.add(commit.get()); - log.warning("No previous tag found for '" + tag.tag() + "'"); } - } else { - commits.addAll(localRepo.commits(previous.get().tag() + ".." + tag.tag()).asList()); } Collections.reverse(commits); + var annotation = localRepo.annotate(tag.tag()); + for (var updater : updaters) { + updater.handleOpenJDKTagCommits(repository, commits, tag, annotation.orElse(null)); + } + } + + var newNonJdkTags = newTags.stream() + .filter(tag -> OpenJDKTag.create(tag).isEmpty()) + .collect(Collectors.toList()); + for (var tag : newNonJdkTags) { + // Update the history first - if there is a problem here we don't want to send out multiple updates + history.addTags(List.of(tag)); + + var commit = localRepo.lookup(tag); + if (commit.isEmpty()) { + throw new RuntimeException("Failed to lookup tag '" + tag.toString() + "'"); + } + + var annotation = localRepo.annotate(tag); for (var updater : updaters) { - updater.handleTagCommits(repository, commits, tag); + updater.handleTagCommit(repository, commit.get(), tag, annotation.orElse(null)); } } } diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JsonUpdater.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JsonUpdater.java index 598633384..6599470c3 100644 --- a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JsonUpdater.java +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JsonUpdater.java @@ -86,7 +86,7 @@ public void handleCommits(HostedRepository repository, List commits, Bra } @Override - public void handleTagCommits(HostedRepository repository, List commits, OpenJDKTag tag) { + public void handleOpenJDKTagCommits(HostedRepository repository, List commits, OpenJDKTag tag, Tag.Annotated annotation) { var build = String.format("b%02d", tag.buildNum()); try (var writer = new JsonUpdateWriter(path, repository.name())) { var issues = new ArrayList(); @@ -100,7 +100,10 @@ public void handleTagCommits(HostedRepository repository, List commits, } @Override - public void handleNewBranch(HostedRepository repository, List commits, Branch parent, Branch branch) { + public void handleTagCommit(HostedRepository repository, Commit commit, Tag tag, Tag.Annotated annotation) { + } + @Override + public void handleNewBranch(HostedRepository repository, List commits, Branch parent, Branch branch) { } } diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/MailingListUpdater.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/MailingListUpdater.java index 65a103a49..e936b3845 100644 --- a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/MailingListUpdater.java +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/MailingListUpdater.java @@ -32,9 +32,9 @@ import java.time.Duration; import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.logging.Logger; public class MailingListUpdater implements UpdateConsumer { private final MailingList list; @@ -77,7 +77,7 @@ private String patchToText(Patch patch) { } } - private String commitToText(HostedRepository repository, Commit commit) { + private String commitToTextBrief(HostedRepository repository, Commit commit) { var writer = new StringWriter(); var printer = new PrintWriter(writer); @@ -88,6 +88,15 @@ private String commitToText(HostedRepository repository, Commit commit) { } printer.println("Date: " + commit.date().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss +0000"))); printer.println("URL: " + repository.webUrl(commit.hash())); + + return writer.toString(); + } + + private String commitToText(HostedRepository repository, Commit commit) { + var writer = new StringWriter(); + var printer = new PrintWriter(writer); + + printer.print(commitToTextBrief(repository, commit)); printer.println(); printer.println(String.join("\n", commit.message())); printer.println(); @@ -101,9 +110,22 @@ private String commitToText(HostedRepository repository, Commit commit) { return writer.toString(); } - private EmailAddress commitsToAuthor(List commits) { - var commit = commits.get(commits.size() - 1); - var commitAddress = EmailAddress.from(commit.committer().name(), commit.committer().email()); + private String tagAnnotationToText(HostedRepository repository, Tag.Annotated annotation) { + var writer = new StringWriter(); + var printer = new PrintWriter(writer); + + printer.println("Tagged by: " + annotation.author().name() + " <" + annotation.author().email() + ">"); + printer.println("Date: " + annotation.date().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss +0000"))); + printer.println(); + printer.print(String.join("\n", annotation.message())); + + return writer.toString(); + } + + private EmailAddress filteredAuthor(EmailAddress commitAddress) { + if (author != null) { + return author; + } var allowedAuthorMatcher = allowedAuthorDomains.matcher(commitAddress.domain()); if (!allowedAuthorMatcher.matches()) { return sender; @@ -112,6 +134,14 @@ private EmailAddress commitsToAuthor(List commits) { } } + private EmailAddress commitToAuthor(Commit commit) { + return filteredAuthor(EmailAddress.from(commit.committer().name(), commit.committer().email())); + } + + private EmailAddress annotationToAuthor(Tag.Annotated annotation) { + return filteredAuthor(EmailAddress.from(annotation.author().name(), annotation.author().email())); + } + private String commitsToSubject(HostedRepository repository, List commits, Branch branch) { var subject = new StringBuilder(); subject.append(repository.repositoryType().shortName()); @@ -131,12 +161,12 @@ private String commitsToSubject(HostedRepository repository, List commit return subject.toString(); } - private String tagToSubject(HostedRepository repository, Hash hash, OpenJDKTag tag) { + private String tagToSubject(HostedRepository repository, Hash hash, Tag tag) { return repository.repositoryType().shortName() + ": " + repository.name() + ": Added tag " + - tag.tag() + + tag + " for changeset " + hash.abbreviate(); } @@ -170,11 +200,11 @@ private List filterAndSendPrCommits(HostedRepository repository, List commi } var subject = commitsToSubject(repository, commits, branch); - var finalAuthor = author != null ? author : commitsToAuthor(commits); + var lastCommit = commits.get(commits.size() - 1); + var commitAddress = filteredAuthor(EmailAddress.from(lastCommit.committer().name(), lastCommit.committer().email())); var email = Email.create(subject, writer.toString()) .sender(sender) - .author(finalAuthor) + .author(commitAddress) .recipient(recipient) .headers(headers) .build(); @@ -224,13 +255,19 @@ public void handleCommits(HostedRepository repository, List commits, Bra } @Override - public void handleTagCommits(HostedRepository repository, List commits, OpenJDKTag tag) { + public void handleOpenJDKTagCommits(HostedRepository repository, List commits, OpenJDKTag tag, Tag.Annotated annotation) { if (mode == Mode.PR_ONLY) { return; } var writer = new StringWriter(); var printer = new PrintWriter(writer); + var taggedCommit = commits.get(commits.size() - 1); + if (annotation != null) { + printer.println(tagAnnotationToText(repository, annotation)); + } + printer.println(commitToTextBrief(repository, taggedCommit)); + printer.println("The following commits are included in " + tag.tag()); printer.println("========================================================"); for (var commit : commits) { @@ -241,17 +278,47 @@ public void handleTagCommits(HostedRepository repository, List commits, printer.println(); } - var tagCommit = commits.get(commits.size() - 1); - var subject = tagToSubject(repository, tagCommit.hash(), tag); - var finalAuthor = author != null ? author : commitsToAuthor(commits); + var subject = tagToSubject(repository, taggedCommit.hash(), tag.tag()); var email = Email.create(subject, writer.toString()) .sender(sender) - .author(finalAuthor) .recipient(recipient) - .headers(headers) - .build(); + .headers(headers); - list.post(email); + if (annotation != null) { + email.author(annotationToAuthor(annotation)); + } else { + email.author(commitToAuthor(taggedCommit)); + } + + list.post(email.build()); + } + + @Override + public void handleTagCommit(HostedRepository repository, Commit commit, Tag tag, Tag.Annotated annotation) { + if (mode == Mode.PR_ONLY) { + return; + } + var writer = new StringWriter(); + var printer = new PrintWriter(writer); + + if (annotation != null) { + printer.println(tagAnnotationToText(repository, annotation)); + } + printer.println(commitToTextBrief(repository, commit)); + + var subject = tagToSubject(repository, commit.hash(), tag); + var email = Email.create(subject, writer.toString()) + .sender(sender) + .recipient(recipient) + .headers(headers); + + if (annotation != null) { + email.author(annotationToAuthor(annotation)); + } else { + email.author(commitToAuthor(commit)); + } + + list.post(email.build()); } private String newBranchSubject(HostedRepository repository, List commits, Branch parent, Branch branch) { @@ -279,7 +346,7 @@ public void handleNewBranch(HostedRepository repository, List commits, B var printer = new PrintWriter(writer); if (commits.size() > 0) { - printer.println("The following commits are unique to the " + branch.name() + " branch"); + printer.println("The following commits are unique to the " + branch.name() + " branch:"); printer.println("========================================================"); for (var commit : commits) { printer.print(commit.hash().abbreviate()); @@ -293,7 +360,8 @@ public void handleNewBranch(HostedRepository repository, List commits, B } var subject = newBranchSubject(repository, commits, parent, branch); - var finalAuthor = author != null ? author : commits.size() > 0 ? commitsToAuthor(commits) : sender; + var finalAuthor = commits.size() > 0 ? commitToAuthor(commits.get(commits.size() - 1)) : sender; + var email = Email.create(subject, writer.toString()) .sender(sender) .author(finalAuthor) diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/UpdateConsumer.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/UpdateConsumer.java index 5dac766bb..a8ece6385 100644 --- a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/UpdateConsumer.java +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/UpdateConsumer.java @@ -30,6 +30,14 @@ public interface UpdateConsumer { void handleCommits(HostedRepository repository, List commits, Branch branch); - void handleTagCommits(HostedRepository repository, List commits, OpenJDKTag tag); + void handleOpenJDKTagCommits(HostedRepository repository, List commits, OpenJDKTag tag, Tag.Annotated annotated); + void handleTagCommit(HostedRepository repository, Commit commit, Tag tag, Tag.Annotated annotation); void handleNewBranch(HostedRepository repository, List commits, Branch parent, Branch branch); + + default void handleOpenJDKTagCommits(HostedRepository repository, List commits, OpenJDKTag tag) { + handleOpenJDKTagCommits(repository, commits, tag, null); + } + default void handleTagCommit(HostedRepository repository, Commit commit, Tag tag) { + handleTagCommit(repository, commit, tag, null); + } } diff --git a/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java b/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java index 868b753e9..43fc81884 100644 --- a/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java +++ b/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java @@ -566,7 +566,7 @@ void testMailinglistTag(TestInfo testInfo) throws IOException { var localRepo = CheckableRepository.init(localRepoFolder, repo.repositoryType()); credentials.commitLock(localRepo); var masterHash = localRepo.resolve("master").orElseThrow(); - localRepo.tag(masterHash, "jdk-12+1", "Added tag 1", "Duke", "duke@openjdk.java.net"); + localRepo.tag(masterHash, "jdk-12+1", "Added tag 1", "Duke Tagger", "tagger@openjdk.java.net"); localRepo.pushAll(repo.url()); var listAddress = EmailAddress.parse(listServer.createList("test")); @@ -591,23 +591,24 @@ void testMailinglistTag(TestInfo testInfo) throws IOException { var editHash = CheckableRepository.appendAndCommit(localRepo, "Another line", "23456789: More fixes"); localRepo.fetch(repo.url(), "history:history"); - localRepo.tag(editHash, "jdk-12+2", "Added tag 2", "Duke", "duke@openjdk.java.net"); + localRepo.tag(editHash, "jdk-12+2", "Added tag 2", "Duke Tagger", "tagger@openjdk.java.net"); CheckableRepository.appendAndCommit(localRepo, "Another line 1", "34567890: Even more fixes"); CheckableRepository.appendAndCommit(localRepo, "Another line 2", "45678901: Yet even more fixes"); var editHash2 = CheckableRepository.appendAndCommit(localRepo, "Another line 3", "56789012: Still even more fixes"); - localRepo.tag(editHash2, "jdk-12+4", "Added tag 3", "Duke", "duke@openjdk.java.net"); + localRepo.tag(editHash2, "jdk-12+4", "Added tag 3", "Duke Tagger", "tagger@openjdk.java.net"); CheckableRepository.appendAndCommit(localRepo, "Another line 4", "67890123: Brand new fixes"); var editHash3 = CheckableRepository.appendAndCommit(localRepo, "Another line 5", "78901234: More brand new fixes"); - localRepo.tag(editHash3, "jdk-13+0", "Added tag 4", "Duke", "duke@openjdk.java.net"); + localRepo.tag(editHash3, "jdk-13+0", "Added tag 4", "Duke Tagger", "tagger@openjdk.java.net"); localRepo.pushAll(repo.url()); TestBotRunner.runPeriodicItems(notifyBot); listServer.processIncoming(); listServer.processIncoming(); listServer.processIncoming(); + listServer.processIncoming(); var conversations = mailmanList.conversations(Duration.ofDays(1)); - assertEquals(3, conversations.size()); + assertEquals(4, conversations.size()); for (var conversation : conversations) { var email = conversation.first(); @@ -618,6 +619,7 @@ void testMailinglistTag(TestInfo testInfo) throws IOException { assertFalse(email.body().contains("56789012: Still even more fixes")); assertFalse(email.body().contains("67890123: Brand new fixes")); assertFalse(email.body().contains("78901234: More brand new fixes")); + assertEquals(EmailAddress.from("Duke Tagger", "tagger@openjdk.java.net"), email.author()); } else if (email.subject().equals("git: test: Added tag jdk-12+4 for changeset " + editHash2.abbreviate())) { assertFalse(email.body().contains("23456789: More fixes")); assertTrue(email.body().contains("34567890: Even more fixes")); @@ -625,6 +627,7 @@ void testMailinglistTag(TestInfo testInfo) throws IOException { assertTrue(email.body().contains("56789012: Still even more fixes")); assertFalse(email.body().contains("67890123: Brand new fixes")); assertFalse(email.body().contains("78901234: More brand new fixes")); + assertEquals(EmailAddress.from("Duke Tagger", "tagger@openjdk.java.net"), email.author()); } else if (email.subject().equals("git: test: Added tag jdk-13+0 for changeset " + editHash3.abbreviate())) { assertFalse(email.body().contains("23456789: More fixes")); assertFalse(email.body().contains("34567890: Even more fixes")); @@ -632,13 +635,15 @@ void testMailinglistTag(TestInfo testInfo) throws IOException { assertFalse(email.body().contains("56789012: Still even more fixes")); assertFalse(email.body().contains("67890123: Brand new fixes")); assertTrue(email.body().contains("78901234: More brand new fixes")); - } else if (email.subject().equals("git: test: 4 new changesets")) { + assertEquals(EmailAddress.from("Duke Tagger", "tagger@openjdk.java.net"), email.author()); + } else if (email.subject().equals("git: test: 6 new changesets")) { assertTrue(email.body().contains("23456789: More fixes")); assertTrue(email.body().contains("34567890: Even more fixes")); assertTrue(email.body().contains("45678901: Yet even more fixes")); assertTrue(email.body().contains("56789012: Still even more fixes")); assertTrue(email.body().contains("67890123: Brand new fixes")); assertTrue(email.body().contains("78901234: More brand new fixes")); + assertEquals(EmailAddress.from("testauthor", "ta@none.none"), email.author()); } else { fail("Mismatched subject: " + email.subject()); } From f3881b629c0ab83e8a553fe13f58524c79cb2cb1 Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Mon, 11 Nov 2019 07:51:16 +0000 Subject: [PATCH 25/54] Add support for offline and local builds Co-authored-by: Nick Gasson Reviewed-by: rwestberg --- README.md | 42 ++++++- bots/cli/build.gradle | 2 +- build.gradle | 73 +++++++++++- .../skara/gradle/images/ImagesPlugin.java | 93 +++++++++++++--- .../skara/gradle/images/LaunchersTask.java | 9 +- .../openjdk/skara/gradle/images/LinkTask.java | 24 +++- cli/build.gradle | 13 ++- deps.env | 12 +- gradlew | 105 ++++++++++-------- gradlew.bat | 6 +- 10 files changed, 285 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 69cd87c17..f934e335a 100644 --- a/README.md +++ b/README.md @@ -39,14 +39,14 @@ external Git source code hosting providers are available: ## Building [JDK 12](http://jdk.java.net/12/) or later and [Gradle](https://gradle.org/) -5.2.1 or later is required for building. To build the project on macOS or -GNU/Linux, just run the following command from the source tree root: +5.6.2 or later is required for building. To build the project on macOS or +GNU/Linux x64, just run the following command from the source tree root: ```bash $ sh gradlew ``` -To build the project on Windows, run the following command from the source tree root: +To build the project on Windows x64, run the following command from the source tree root: ```bat > gradlew @@ -55,6 +55,42 @@ To build the project on Windows, run the following command from the source tree The extracted jlinked image will end up in the `build` directory in the source tree root. +### Other operating systems and CPU architectures + +If you want to build on an operating system other than GNU/Linux, macOS or +Windows _or_ if you want to build on a CPU architecture other than x64, then +ensure that you have JDK 12 or later installed locally. You can then run the +following command from the source tree root: + +```bash +$ sh gradlew +``` + +The extracted jlinked image will end up in the `build` directory in the source +tree root. + +### Offline builds + +If you don't want the build to automatically download any dependencies, then +you must ensure that you have installed the following software locally: + +- JDK 12 or later +- Gradle 5.6.2 or later + +To create a build then run the command: + +```bash +$ gradle offline +``` + +_Please note_ that the above command does _not_ make use of `gradlew` to avoid +downloading Gradle. + +The extracted jlinked image will end up in the `build` directory in the source +tree root. + +### Cross-linking + It is also supported to cross-jlink jimages to GNU/Linux, macOS and/or Windows from any of the aforementioned operating systems. To build all applicable jimages (including the server-side tooling), run the following command from the diff --git a/bots/cli/build.gradle b/bots/cli/build.gradle index 71c3be055..6c2c3ee1b 100644 --- a/bots/cli/build.gradle +++ b/bots/cli/build.gradle @@ -64,7 +64,7 @@ dependencies { } images { - linux { + linux_x64 { modules = ['jdk.crypto.ec', 'org.openjdk.skara.bots.pr', 'org.openjdk.skara.bots.hgbridge', diff --git a/build.gradle b/build.gradle index 350d23f6b..dbef0890f 100644 --- a/build.gradle +++ b/build.gradle @@ -104,19 +104,82 @@ reproduce { dockerfile = 'test.dockerfile' } +def getOS() { + def os = System.getProperty('os.name').toLowerCase() + if (os.startsWith('linux')) { + return 'linux' + } + if (os.startsWith('mac')) { + return 'macos' + } + if (os.startsWith('win')) { + return 'windows' + } + if (os.startsWith('sunos')) { + return 'solaris' + } + throw new GradleException("Unexpected operating system: " + os) +} + +def getCPU() { + def cpu = System.getProperty('os.arch').toLowerCase() + if (cpu.startsWith('amd64') || cpu.startsWith('x86_64') || cpu.startsWith('x64')) { + return 'x64' + } + if (cpu.startsWith('x86') || cpu.startsWith('i386')) { + return 'x86' + } + if (cpu.startsWith('sparc')) { + return 'sparc' + } + if (cpu.startsWith('ppc')) { + return 'ppc' + } + if (cpu.startsWith('arm')) { + return 'arm' + } + if (cpu.startsWith('aarch64')) { + return 'aarch64'; + } + throw new GradleException("Unexpected CPU: " + cpu) +} + task local(type: Copy) { doFirst { delete project.buildDir } - def os = System.getProperty('os.name').toLowerCase() - def osName = os.startsWith('win') ? 'Windows' : - os.startsWith('mac') ? 'Macos' : 'Linux' - dependsOn ':cli:image' + osName + def os = getOS() + def cpu = getCPU() + + if (os in ['linux', 'macos', 'windows'] && cpu == 'x64') { + def target = os.substring(0, 1).toUpperCase() + os.substring(1) + + cpu.substring(0, 1).toUpperCase() + cpu.substring(1) + dependsOn ':cli:image' + target + } else { + dependsOn ':cli:imageLocal' + } + + from zipTree(file(project.rootDir.toString() + + '/cli/build/distributions/cli' + + '-' + project.version + '-' + + os + '-' + cpu + '.zip')) + into project.buildDir +} + +task offline(type: Copy) { + doFirst { + delete project.buildDir + } + + def os = getOS() + def cpu = getCPU() + + dependsOn ':cli:imageLocal' from zipTree(file(project.rootDir.toString() + '/cli/build/distributions/cli' + '-' + project.version + '-' + - osName.toLowerCase() + '.zip')) + os + '-' + cpu + '.zip')) into project.buildDir } diff --git a/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/ImagesPlugin.java b/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/ImagesPlugin.java index dbb808be7..01b9c2a69 100644 --- a/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/ImagesPlugin.java +++ b/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/ImagesPlugin.java @@ -29,9 +29,52 @@ import org.gradle.api.artifacts.UnknownConfigurationException; import java.util.ArrayList; +import java.util.HashSet; import java.io.File; public class ImagesPlugin implements Plugin { + private static String getOS() { + var p = System.getProperty("os.name").toLowerCase(); + if (p.startsWith("win")) { + return "windows"; + } + if (p.startsWith("mac")) { + return "macos"; + } + if (p.startsWith("linux")) { + return "linux"; + } + if (p.startsWith("sunos")) { + return "solaris"; + } + + throw new RuntimeException("Unknown operating system: " + System.getProperty("os.name")); + } + + private static String getCPU() { + var p = System.getProperty("os.arch").toLowerCase(); + if (p.startsWith("amd64") || p.startsWith("x86_64") || p.startsWith("x64")) { + return "x64"; + } + if (p.startsWith("x86") || p.startsWith("i386")) { + return "x86"; + } + if (p.startsWith("sparc")) { + return "sparc"; + } + if (p.startsWith("ppc")) { + return "ppc"; + } + if (p.startsWith("arm")) { + return "arm"; + } + if (p.startsWith("aarch64")) { + return "aarch64"; + } + + throw new RuntimeException("Unknown CPU: " + System.getProperty("os.arch")); + } + @Override public void apply(Project project) { NamedDomainObjectContainer imageEnvironmentContainer = @@ -49,15 +92,23 @@ public ImageEnvironment create(String name) { imageEnvironmentContainer.all(new Action() { public void execute(ImageEnvironment env) { - var name = env.getName(); - var subName = name.substring(0, 1).toUpperCase() + name.substring(1); + var parts = env.getName().split("_");; + var isLocal = parts.length == 1 && parts[0].equals("local"); + var os = isLocal ? getOS() : parts[0]; + var cpu = isLocal ? getCPU() : parts[1]; + var osAndCpuPascalCased = + os.substring(0, 1).toUpperCase() + os.substring(1) + + cpu.substring(0, 1).toUpperCase() + cpu.substring(1); + var subName = isLocal ? "Local" : osAndCpuPascalCased; var downloadTaskName = "download" + subName + "JDK"; - project.getTasks().register(downloadTaskName, DownloadJDKTask.class, (task) -> { - task.getUrl().set(env.getUrl()); - task.getSha256().set(env.getSha256()); - task.getToDir().set(rootDir.resolve(".jdk")); - }); + if (!isLocal) { + project.getTasks().register(downloadTaskName, DownloadJDKTask.class, (task) -> { + task.getUrl().set(env.getUrl()); + task.getSha256().set(env.getSha256()); + task.getToDir().set(rootDir.resolve(".jdk")); + }); + } var linkTaskName = "link" + subName; project.getTasks().register(linkTaskName, LinkTask.class, (task) -> { @@ -75,10 +126,15 @@ public void execute(ImageEnvironment env) { // ignored } - task.dependsOn(projectPath + ":" + downloadTaskName); + if (!isLocal) { + task.dependsOn(projectPath + ":" + downloadTaskName); + task.getUrl().set(env.getUrl()); + } else { + task.getUrl().set("local"); + } task.getToDir().set(buildDir.resolve("images")); - task.getUrl().set(env.getUrl()); - task.getOS().set(name); + task.getOS().set(os); + task.getCPU().set(cpu); task.getLaunchers().set(env.getLaunchers()); task.getModules().set(env.getModules()); }); @@ -88,7 +144,8 @@ public void execute(ImageEnvironment env) { task.getLaunchers().set(env.getLaunchers()); task.getOptions().set(env.getOptions()); task.getToDir().set(buildDir.resolve("launchers")); - task.getOS().set(name); + task.getOS().set(os); + task.getCPU().set(cpu); }); var zipTaskName = "bundleZip" + subName; @@ -99,7 +156,7 @@ public void execute(ImageEnvironment env) { task.setPreserveFileTimestamps(false); task.setReproducibleFileOrder(true); task.getArchiveBaseName().set(project.getName()); - task.getArchiveClassifier().set(name); + task.getArchiveClassifier().set(os + "-" + cpu); task.getArchiveExtension().set("zip"); if (env.getMan().isPresent()) { @@ -109,10 +166,11 @@ public void execute(ImageEnvironment env) { }); } - task.from(buildDir.resolve("images").resolve(name), (s) -> { + var subdir = os + "-" + cpu; + task.from(buildDir.resolve("images").resolve(subdir), (s) -> { s.into("image"); }); - task.from(buildDir.resolve("launchers").resolve(name), (s) -> { + task.from(buildDir.resolve("launchers").resolve(subdir), (s) -> { s.into("bin"); }); }); @@ -125,7 +183,7 @@ public void execute(ImageEnvironment env) { task.setPreserveFileTimestamps(false); task.setReproducibleFileOrder(true); task.getArchiveBaseName().set(project.getName()); - task.getArchiveClassifier().set(name); + task.getArchiveClassifier().set(os + "-" + cpu); task.getArchiveExtension().set("tar.gz"); task.setCompression(Compression.GZIP); @@ -136,10 +194,11 @@ public void execute(ImageEnvironment env) { }); } - task.from(buildDir.resolve("images").resolve(name), (s) -> { + var subdir = os + "-" + cpu; + task.from(buildDir.resolve("images").resolve(subdir), (s) -> { s.into("image"); }); - task.from(buildDir.resolve("launchers").resolve(name), (s) -> { + task.from(buildDir.resolve("launchers").resolve(subdir), (s) -> { s.into("bin"); }); }); diff --git a/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LaunchersTask.java b/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LaunchersTask.java index 59f13e2da..85339669b 100644 --- a/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LaunchersTask.java +++ b/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LaunchersTask.java @@ -37,6 +37,7 @@ public class LaunchersTask extends DefaultTask { private Property toDir; private Property os; + private Property cpu; private MapProperty launchers; private ListProperty options; @@ -44,6 +45,7 @@ public class LaunchersTask extends DefaultTask { public LaunchersTask(ObjectFactory factory) { toDir = factory.property(Path.class); os = factory.property(String.class); + cpu = factory.property(String.class); launchers = factory.mapProperty(String.class, String.class); options = factory.listProperty(String.class); } @@ -63,6 +65,11 @@ Property getOS() { return os; } + @Input + Property getCPU() { + return cpu; + } + @Input MapProperty getLaunchers() { return launchers; @@ -77,7 +84,7 @@ private static void clearDirectory(Path directory) throws IOException { @TaskAction void generate() throws IOException { - var dest = toDir.get().resolve(os.get()); + var dest = toDir.get().resolve(os.get() + "-" + cpu.get()); if (Files.isDirectory(dest)) { clearDirectory(dest); } diff --git a/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LinkTask.java b/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LinkTask.java index 6563e8db8..1cd6a8b7a 100644 --- a/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LinkTask.java +++ b/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LinkTask.java @@ -41,6 +41,7 @@ public class LinkTask extends DefaultTask { private final Property os; + private final Property cpu; private final Property url; private final Property toDir; private final MapProperty launchers; @@ -51,6 +52,7 @@ public class LinkTask extends DefaultTask { @Inject public LinkTask(ObjectFactory factory) { os = factory.property(String.class); + cpu = factory.property(String.class); url = factory.property(String.class); toDir = factory.property(Path.class); launchers = factory.mapProperty(String.class, String.class); @@ -69,6 +71,11 @@ Property getOS() { return os; } + @Input + Property getCPU() { + return cpu; + } + @Input Property getUrl() { return url; @@ -117,9 +124,14 @@ void link() throws IOException { modularJars.add(jar.getAsFile().toString()); } - var filename = Path.of(URI.create(url.get()).getPath()).getFileName().toString(); - var dirname = filename.replace(".zip", "").replace(".tar.gz", ""); - var jdk = project.getRootDir().toPath().toAbsolutePath().resolve(".jdk").resolve(dirname); + Path jdk = null; + if (!url.get().equals("local")) { + var filename = Path.of(URI.create(url.get()).getPath()).getFileName().toString(); + var dirname = filename.replace(".zip", "").replace(".tar.gz", ""); + jdk = project.getRootDir().toPath().toAbsolutePath().resolve(".jdk").resolve(dirname); + } else { + jdk = Path.of(System.getProperty("java.home")); + } var dirs = Files.walk(jdk) .filter(Files::isDirectory) .filter(p -> p.getFileName().toString().equals("jmods")) @@ -143,7 +155,7 @@ void link() throws IOException { var allModules = new ArrayList(uniqueModules); Files.createDirectories(toDir.get()); - var dest = toDir.get().resolve(os.get()); + var dest = toDir.get().resolve(os.get() + "-" + cpu.get()); if (Files.exists(dest) && Files.isDirectory(dest)) { clearDirectory(dest); } @@ -163,14 +175,14 @@ void link() throws IOException { }); var currentOS = System.getProperty("os.name").toLowerCase().substring(0, 3); - if (currentOS.equals(os.get().substring(0, 3))) { + if (os.get().equals("local") || currentOS.equals(os.get().substring(0, 3))) { var ext = currentOS.startsWith("win") ? ".exe" : ""; var javaLaunchers = Files.walk(dest) .filter(Files::isExecutable) .filter(p -> p.getFileName().toString().equals("java" + ext)) .collect(Collectors.toList()); if (javaLaunchers.size() != 1) { - throw new GradleException("Multiple or no java launchers generated for " + os.get() + " image"); + throw new GradleException("Multiple or no java launchers generated for " + os.get() + "-" + cpu.get() + " image"); } var java = javaLaunchers.get(0); project.exec((spec) -> { diff --git a/cli/build.gradle b/cli/build.gradle index 9a9b4c3ac..41a2053fc 100644 --- a/cli/build.gradle +++ b/cli/build.gradle @@ -71,7 +71,7 @@ images { ext.modules = ['jdk.crypto.ec'] - windows { + windows_x64 { modules = ext.modules launchers = ext.launchers bundles = ['zip', 'tar.gz'] @@ -81,7 +81,7 @@ images { } } - linux { + linux_x64 { modules = ext.modules launchers = ext.launchers man = 'cli/resources/man' @@ -92,7 +92,7 @@ images { } } - macos { + macos_x64 { modules = ext.modules launchers = ext.launchers man = 'cli/resources/man' @@ -102,4 +102,11 @@ images { sha256 = '52164a04db4d3fdfe128cfc7b868bc4dae52d969f03d53ae9d4239fe783e1a3a' } } + + local { + modules = ext.modules + launchers = ext.launchers + man = 'cli/resources/man' + bundles = ['zip', 'tar.gz'] + } } diff --git a/deps.env b/deps.env index 09ca6d999..b58b33f01 100644 --- a/deps.env +++ b/deps.env @@ -1,11 +1,11 @@ -JDK_LINUX_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_linux-x64_bin.tar.gz" -JDK_LINUX_SHA256="b43bc15f4934f6d321170419f2c24451486bc848a2179af5e49d10721438dd56" +JDK_LINUX_X64_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_linux-x64_bin.tar.gz" +JDK_LINUX_X64_SHA256="b43bc15f4934f6d321170419f2c24451486bc848a2179af5e49d10721438dd56" -JDK_MACOS_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_osx-x64_bin.tar.gz" -JDK_MACOS_SHA256="52164a04db4d3fdfe128cfc7b868bc4dae52d969f03d53ae9d4239fe783e1a3a" +JDK_MACOS_X64_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_osx-x64_bin.tar.gz" +JDK_MACOS_X64_SHA256="52164a04db4d3fdfe128cfc7b868bc4dae52d969f03d53ae9d4239fe783e1a3a" -JDK_WINDOWS_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_windows-x64_bin.zip" -JDK_WINDOWS_SHA256="35a8d018f420fb05fe7c2aa9933122896ca50bd23dbd373e90d8e2f3897c4e92" +JDK_WINDOWS_X64_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_windows-x64_bin.zip" +JDK_WINDOWS_X64_SHA256="35a8d018f420fb05fe7c2aa9933122896ca50bd23dbd373e90d8e2f3897c4e92" GRADLE_URL="https://services.gradle.org/distributions/gradle-5.6.2-bin.zip" GRADLE_SHA256="32fce6628848f799b0ad3205ae8db67d0d828c10ffe62b748a7c0d9f4a5d9ee0" diff --git a/gradlew b/gradlew index bb95ec1f3..b36faf64b 100644 --- a/gradlew +++ b/gradlew @@ -81,68 +81,71 @@ extract_zip() { } DIR=$(dirname $0) +ARCH=$(uname -m) OS=$(uname) . $(dirname "${0}")/deps.env -case "${OS}" in - Linux ) - JDK_URL="${JDK_LINUX_URL}" - JDK_SHA256="${JDK_LINUX_SHA256}" - ;; - Darwin ) - JDK_URL="${JDK_MACOS_URL}" - JDK_SHA256="${JDK_MACOS_SHA256}" - ;; - CYGWIN_NT* ) - JDK_URL="${JDK_WINDOWS_URL}" - JDK_SHA256="${JDK_WINDOWS_SHA256}" - ;; - *) - echo "error: unknown operating system ${OS}" - exit 1 - ;; -esac - -JDK_FILENAME="${DIR}/.jdk/$(basename ${JDK_URL})" -if [ "${OS}" = "Linux" -o "${OS}" = "Darwin" ]; then - JDK_DIR="${DIR}/.jdk/$(basename -s '.tar.gz' ${JDK_URL})" -else - JDK_DIR="${DIR}/.jdk/$(basename -s '.zip' ${JDK_URL})" +if [ "${ARCH}" = "x86_64" ]; then + case "${OS}" in + Linux ) + JDK_URL="${JDK_LINUX_X64_URL}" + JDK_SHA256="${JDK_LINUX_X64_SHA256}" + ;; + Darwin ) + JDK_URL="${JDK_MACOS_X64_URL}" + JDK_SHA256="${JDK_MACOS_X64_SHA256}" + ;; + CYGWIN_NT* ) + JDK_URL="${JDK_WINDOWS_X64_URL}" + JDK_SHA256="${JDK_WINDOWS_X64_SHA256}" + ;; + esac fi -if [ ! -d "${JDK_DIR}" ]; then - mkdir -p ${DIR}/.jdk - if [ ! -f "${JDK_FILENAME}" ]; then - if [ -f "${JDK_URL}" ]; then - echo "Copying JDK..." - cp "${JDK_URL}" "${JDK_FILENAME}" +if [ ! -z "${JDK_URL}" ]; then + JDK_FILENAME="${DIR}/.jdk/$(basename ${JDK_URL})" + if [ "${OS}" = "Linux" -o "${OS}" = "Darwin" ]; then + JDK_DIR="${DIR}/.jdk/$(basename -s '.tar.gz' ${JDK_URL})" + else + JDK_DIR="${DIR}/.jdk/$(basename -s '.zip' ${JDK_URL})" + fi + + if [ ! -d "${JDK_DIR}" ]; then + mkdir -p ${DIR}/.jdk + if [ ! -f "${JDK_FILENAME}" ]; then + if [ -f "${JDK_URL}" ]; then + echo "Copying JDK..." + cp "${JDK_URL}" "${JDK_FILENAME}" + else + echo "Downloading JDK..." + download ${JDK_URL} "${JDK_FILENAME}" + checksum "${JDK_FILENAME}" ${JDK_SHA256} + fi + fi + echo "Extracting JDK..." + if [ "${OS}" = "Linux" -o "${OS}" = "Darwin" ]; then + extract_tar "${JDK_FILENAME}" "${JDK_DIR}" else - echo "Downloading JDK..." - download ${JDK_URL} "${JDK_FILENAME}" - checksum "${JDK_FILENAME}" ${JDK_SHA256} + extract_zip "${JDK_FILENAME}" "${JDK_DIR}" fi fi - echo "Extracting JDK..." - if [ "${OS}" = "Linux" -o "${OS}" = "Darwin" ]; then - extract_tar "${JDK_FILENAME}" "${JDK_DIR}" + + if [ "${OS}" = "Darwin" ]; then + EXECUTABLE_FILTER='-perm +111' + LAUNCHER='java' + elif [ "${OS}" = "Linux" ]; then + EXECUTABLE_FILTER='-executable' + LAUNCHER='java' else - extract_zip "${JDK_FILENAME}" "${JDK_DIR}" + LAUNCHER='java.exe' fi -fi -if [ "${OS}" = "Darwin" ]; then - EXECUTABLE_FILTER='-perm +111' - LAUNCHER='java' -elif [ "${OS}" = "Linux" ]; then - EXECUTABLE_FILTER='-executable' - LAUNCHER='java' + JAVA_LAUNCHER=$(find "${JDK_DIR}" -type f ${EXECUTABLE_FILTER} | grep ".*/bin/${LAUNCHER}$") + export JAVA_HOME="$(dirname $(dirname ${JAVA_LAUNCHER}))" else - LAUNCHER='java.exe' + JAVA_LAUNCHER="java" fi -JAVA_LAUNCHER=$(find "${JDK_DIR}" -type f ${EXECUTABLE_FILTER} | grep ".*/bin/${LAUNCHER}$") -export JAVA_HOME="$(dirname $(dirname ${JAVA_LAUNCHER}))" - GRADLE_FILENAME="${DIR}/.gradle/$(basename ${GRADLE_URL})" GRADLE_DIR="${DIR}/.gradle/$(basename -s '.zip' ${GRADLE_URL})" @@ -155,7 +158,11 @@ if [ ! -d "${GRADLE_DIR}" ]; then checksum ${GRADLE_FILENAME} ${GRADLE_SHA256} echo "Extracting Gradle..." if [ "${OS}" = "Linux" -o "${OS}" = "Darwin" ]; then - "${JAVA_LAUNCHER}" "${DIR}"/Unzip.java "${GRADLE_FILENAME}" "${GRADLE_DIR}" + if exists unzip; then + extract_zip "${GRADLE_FILENAME}" "${GRADLE_DIR}" + else + "${JAVA_LAUNCHER}" "${DIR}"/Unzip.java "${GRADLE_FILENAME}" "${GRADLE_DIR}" + fi else extract_zip "${GRADLE_FILENAME}" "${GRADLE_DIR}" fi diff --git a/gradlew.bat b/gradlew.bat index f1842dd5e..3fa9fad29 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -21,17 +21,17 @@ rem or visit www.oracle.com if you need additional information or have any rem questions. for /f "tokens=1,2 delims==" %%A in (deps.env) do (set %%A=%%~B) -for /f %%i in ("%JDK_WINDOWS_URL%") do set JDK_WINDOWS_DIR=%%~ni +for /f %%i in ("%JDK_WINDOWS_X64_URL%") do set JDK_WINDOWS_DIR=%%~ni for /f %%i in ("%GRADLE_URL%") do set GRADLE_DIR=%%~ni if exist %~dp0\.jdk\%JDK_WINDOWS_DIR% goto gradle echo Downloading JDK... mkdir %~dp0\.jdk\temp -curl -L %JDK_WINDOWS_URL% -o %JDK_WINDOWS_DIR%.zip +curl -L %JDK_WINDOWS_X64_URL% -o %JDK_WINDOWS_DIR%.zip move %JDK_WINDOWS_DIR%.zip %~dp0\.jdk\ for /f "tokens=*" %%i in ('@certutil -hashfile %~dp0/.jdk/%JDK_WINDOWS_DIR%.zip sha256 ^| %WINDIR%\System32\find /v "hash of file" ^| %WINDIR%\System32\find /v "CertUtil"') do set SHA256JDK=%%i -if "%SHA256JDK%" == "%JDK_WINDOWS_SHA256%" (goto extractJdk) +if "%SHA256JDK%" == "%JDK_WINDOWS_X64_SHA256%" (goto extractJdk) echo Invalid SHA256 for JDK detected (%SHA256JDK%) goto done From 94ccf84115a30b51324418ded990117ea795084b Mon Sep 17 00:00:00 2001 From: Jorn Vernee Date: Mon, 11 Nov 2019 09:23:15 +0000 Subject: [PATCH 26/54] 153: Check if origin remote exists before trying to get the pullPath Reviewed-by: ehelin --- .../java/org/openjdk/skara/cli/GitWebrev.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cli/src/main/java/org/openjdk/skara/cli/GitWebrev.java b/cli/src/main/java/org/openjdk/skara/cli/GitWebrev.java index a21a85eb2..711e8f861 100644 --- a/cli/src/main/java/org/openjdk/skara/cli/GitWebrev.java +++ b/cli/src/main/java/org/openjdk/skara/cli/GitWebrev.java @@ -156,15 +156,17 @@ private static void generate(String[] args) throws IOException { if (upstream == null) { try { var remote = isMercurial ? "default" : "origin"; - var pullPath = repo.pullPath(remote); - var uri = new URI(pullPath); - var host = uri.getHost(); - var path = uri.getPath(); - if (host != null && path != null) { - if (host.equals("github.com") && path.startsWith("/openjdk/")) { - upstream = "https://github.com" + path; - } else if (host.equals("openjdk.java.net")) { - upstream = "https://openjdk.java.net" + path; + if (repo.remotes().contains(remote)) { + var pullPath = repo.pullPath(remote); + var uri = new URI(pullPath); + var host = uri.getHost(); + var path = uri.getPath(); + if (host != null && path != null) { + if (host.equals("github.com") && path.startsWith("/openjdk/")) { + upstream = "https://github.com" + path; + } else if (host.equals("openjdk.java.net")) { + upstream = "https://openjdk.java.net" + path; + } } } } catch (URISyntaxException e) { From 49bbd41867e743113ea1065cdf300294006c3fa9 Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Mon, 11 Nov 2019 13:23:45 +0000 Subject: [PATCH 27/54] Update to Gradle 6.0 Reviewed-by: rwestberg --- buildSrc/build.gradle | 2 +- cli/build.gradle | 2 +- deps.env | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 493965582..b2f2bca71 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -26,7 +26,7 @@ plugins { } dependencies { - runtime subprojects.collect { owner.project(it.path) } + runtimeOnly subprojects.collect { owner.project(it.path) } } defaultTasks 'compileJava' diff --git a/cli/build.gradle b/cli/build.gradle index 41a2053fc..9da0a98ec 100644 --- a/cli/build.gradle +++ b/cli/build.gradle @@ -47,7 +47,7 @@ dependencies { jar { manifest { - attributes("Implementation-Title": "org.openjdk.skara.cli", "Implementation-Version": version) + attributes("Implementation-Title": "org.openjdk.skara.cli", "Implementation-Version": archiveVersion) } } diff --git a/deps.env b/deps.env index b58b33f01..fc6008567 100644 --- a/deps.env +++ b/deps.env @@ -7,5 +7,5 @@ JDK_MACOS_X64_SHA256="52164a04db4d3fdfe128cfc7b868bc4dae52d969f03d53ae9d4239fe78 JDK_WINDOWS_X64_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_windows-x64_bin.zip" JDK_WINDOWS_X64_SHA256="35a8d018f420fb05fe7c2aa9933122896ca50bd23dbd373e90d8e2f3897c4e92" -GRADLE_URL="https://services.gradle.org/distributions/gradle-5.6.2-bin.zip" -GRADLE_SHA256="32fce6628848f799b0ad3205ae8db67d0d828c10ffe62b748a7c0d9f4a5d9ee0" +GRADLE_URL="https://services.gradle.org/distributions/gradle-6.0-bin.zip" +GRADLE_SHA256="5a3578b9f0bb162f5e08cf119f447dfb8fa950cedebb4d2a977e912a11a74b91" From 2a8f113cc792746c80a8bf4ddeb53c91d2cccb50 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Mon, 11 Nov 2019 14:19:15 +0000 Subject: [PATCH 28/54] Bridgekeeper bot Reviewed-by: ehelin --- bots/bridgekeeper/build.gradle | 44 +++++++ .../src/main/java/module-info.java | 30 +++++ .../bots/bridgekeeper/BridgekeeperBot.java | 118 ++++++++++++++++++ .../bridgekeeper/BridgekeeperBotFactory.java | 47 +++++++ .../bridgekeeper/BridgekeeperBotTests.java | 99 +++++++++++++++ settings.gradle | 1 + 6 files changed, 339 insertions(+) create mode 100644 bots/bridgekeeper/build.gradle create mode 100644 bots/bridgekeeper/src/main/java/module-info.java create mode 100644 bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBot.java create mode 100644 bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotFactory.java create mode 100644 bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotTests.java diff --git a/bots/bridgekeeper/build.gradle b/bots/bridgekeeper/build.gradle new file mode 100644 index 000000000..a2182ecb6 --- /dev/null +++ b/bots/bridgekeeper/build.gradle @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +module { + name = 'org.openjdk.skara.bots.bridgekeeper' + test { + requires 'org.junit.jupiter.api' + requires 'org.openjdk.skara.vcs' + requires 'org.openjdk.skara.test' + opens 'org.openjdk.skara.bots.bridgekeeper' to 'org.junit.platform.commons' + } +} + +dependencies { + implementation project(':host') + implementation project(':forge') + implementation project(':issuetracker') + implementation project(':bot') + implementation project(':census') + implementation project(':json') + implementation project(':vcs') + + testImplementation project(':test') +} diff --git a/bots/bridgekeeper/src/main/java/module-info.java b/bots/bridgekeeper/src/main/java/module-info.java new file mode 100644 index 000000000..6e425b9aa --- /dev/null +++ b/bots/bridgekeeper/src/main/java/module-info.java @@ -0,0 +1,30 @@ +import org.openjdk.skara.bots.bridgekeeper.BridgekeeperBotFactory; + +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +module org.openjdk.skara.bots.bridgekeeper { + requires org.openjdk.skara.bot; + requires java.logging; + + provides org.openjdk.skara.bot.BotFactory with BridgekeeperBotFactory; +} diff --git a/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBot.java b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBot.java new file mode 100644 index 000000000..d6210a702 --- /dev/null +++ b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBot.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.bridgekeeper; + +import org.openjdk.skara.bot.*; +import org.openjdk.skara.forge.*; + +import java.nio.file.Path; +import java.util.*; +import java.util.function.Consumer; +import java.util.logging.Logger; + +class BridgekeeperWorkItem implements WorkItem { + private final Logger log = Logger.getLogger("org.openjdk.skara.bots");; + private final HostedRepository repository; + private final PullRequest pr; + private final Consumer errorHandler; + + BridgekeeperWorkItem(HostedRepository repository, PullRequest pr, Consumer errorHandler) { + this.pr = pr; + this.repository = repository; + this.errorHandler = errorHandler; + } + + private final String welcomeMarker = ""; + + private void checkWelcomeMessage() { + log.info("Checking welcome message of " + pr); + + var comments = pr.comments(); + var welcomePosted = comments.stream() + .anyMatch(comment -> comment.body().contains(welcomeMarker)); + + if (!welcomePosted) { + var message = "Welcome to the OpenJDK organization on GitHub!\n\n" + + "This repository is currently a read-only git mirror of the official Mercurial " + + "repository (located at https://hg.openjdk.java.net/). As such, we are not " + + "currently accepting pull requests here. If you would like to contribute to " + + "the OpenJDK project, please see http://openjdk.java.net/contribute/ on how " + + "to proceed.\n\n" + + "This pull request will be automatically closed."; + + log.fine("Posting welcome message"); + pr.addComment(welcomeMarker + "\n\n" + message); + } + pr.setState(PullRequest.State.CLOSED); + } + + + @Override + public boolean concurrentWith(WorkItem other) { + if (!(other instanceof BridgekeeperWorkItem)) { + return true; + } + BridgekeeperWorkItem otherItem = (BridgekeeperWorkItem)other; + if (!pr.id().equals(otherItem.pr.id())) { + return true; + } + if (!repository.name().equals(otherItem.repository.name())) { + return true; + } + return false; + } + + @Override + public void run(Path scratchPath) { + checkWelcomeMessage(); + } + + @Override + public void handleRuntimeException(RuntimeException e) { + errorHandler.accept(e); + } +} + +public class BridgekeeperBot implements Bot { + private final HostedRepository remoteRepo; + private final PullRequestUpdateCache updateCache; + + BridgekeeperBot(HostedRepository repo) { + this.remoteRepo = repo; + this.updateCache = new PullRequestUpdateCache(); + } + + @Override + public List getPeriodicItems() { + List ret = new LinkedList<>(); + + for (var pr : remoteRepo.pullRequests()) { + if (updateCache.needsUpdate(pr)) { + var item = new BridgekeeperWorkItem(remoteRepo, pr, e -> updateCache.invalidate(pr)); + ret.add(item); + } + } + + return ret; + } +} diff --git a/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotFactory.java b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotFactory.java new file mode 100644 index 000000000..50d8c2b10 --- /dev/null +++ b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotFactory.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.bridgekeeper; + +import org.openjdk.skara.bot.*; + +import java.util.*; + +public class BridgekeeperBotFactory implements BotFactory { + @Override + public String name() { + return "bridgekeeper"; + } + + @Override + public List create(BotConfiguration configuration) { + var ret = new ArrayList(); + var specific = configuration.specific(); + + for (var repo : specific.get("repositories").asArray()) { + var bot = new BridgekeeperBot(configuration.repository(repo.asString())); + ret.add(bot); + } + + return ret; + } +} diff --git a/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotTests.java b/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotTests.java new file mode 100644 index 000000000..ebc7b8f59 --- /dev/null +++ b/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotTests.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.bridgekeeper; + +import org.openjdk.skara.issuetracker.Issue; +import org.openjdk.skara.test.*; + +import org.junit.jupiter.api.*; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class BridgekeeperBotTests { + @Test + void simple(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var bot = new BridgekeeperBot(author); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Let the bot see it + TestBotRunner.runPeriodicItems(bot); + + // There should now be no open PRs + var prs = author.pullRequests(); + assertEquals(0, prs.size()); + } + } + + @Test + void keepClosing(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var bot = new BridgekeeperBot(author); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Let the bot see it + TestBotRunner.runPeriodicItems(bot); + + // There should now be no open PRs + var prs = author.pullRequests(); + assertEquals(0, prs.size()); + + // The author is persistent + pr.setState(Issue.State.OPEN); + prs = author.pullRequests(); + assertEquals(1, prs.size()); + + // But so is the bot + TestBotRunner.runPeriodicItems(bot); + prs = author.pullRequests(); + assertEquals(0, prs.size()); + + // There should still only be one welcome comment + assertEquals(1, pr.comments().size()); + } + } +} diff --git a/settings.gradle b/settings.gradle index 54bbef206..f2b790b16 100644 --- a/settings.gradle +++ b/settings.gradle @@ -44,6 +44,7 @@ include 'network' include 'forge' include 'issuetracker' +include 'bots:bridgekeeper' include 'bots:cli' include 'bots:forward' include 'bots:hgbridge' From 980b7a8742be8f8f3fccf86c1efb523593e09f76 Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Tue, 12 Nov 2019 09:01:57 +0000 Subject: [PATCH 29/54] Add tester bot and ci module Reviewed-by: rwestberg --- bot/build.gradle | 1 + bot/src/main/java/module-info.java | 1 + .../openjdk/skara/bot/BotConfiguration.java | 8 + .../skara/bot/BotRunnerConfiguration.java | 31 + bots/bridgekeeper/build.gradle | 1 + bots/cli/build.gradle | 3 + bots/forward/build.gradle | 1 + bots/hgbridge/build.gradle | 1 + bots/merge/build.gradle | 1 + bots/mirror/build.gradle | 1 + bots/mlbridge/build.gradle | 1 + bots/notify/build.gradle | 1 + bots/pr/build.gradle | 1 + bots/submit/build.gradle | 1 + bots/tester/build.gradle | 45 + bots/tester/src/main/java/module-info.java | 31 + .../org/openjdk/skara/bots/tester/Stage.java | 34 + .../org/openjdk/skara/bots/tester/State.java | 166 +++ .../openjdk/skara/bots/tester/TestBot.java | 106 ++ .../skara/bots/tester/TestBotFactory.java | 66 ++ .../skara/bots/tester/TestWorkItem.java | 443 ++++++++ .../tester/InMemoryContinuousIntegration.java | 99 ++ .../skara/bots/tester/InMemoryHost.java | 63 ++ .../bots/tester/InMemoryHostedRepository.java | 132 +++ .../skara/bots/tester/InMemoryJob.java | 67 ++ .../bots/tester/InMemoryPullRequest.java | 230 +++++ .../openjdk/skara/bots/tester/StateTests.java | 391 +++++++ .../skara/bots/tester/TestBotTests.java | 61 ++ .../skara/bots/tester/TestWorkItemTests.java | 952 ++++++++++++++++++ bots/topological/build.gradle | 1 + ci/build.gradle | 36 + ci/src/main/java/module-info.java | 29 + .../main/java/org/openjdk/skara/ci/Build.java | 106 ++ .../skara/ci/ContinuousIntegration.java | 53 + .../ci/ContinuousIntegrationFactory.java | 38 + .../main/java/org/openjdk/skara/ci/Job.java | 105 ++ .../main/java/org/openjdk/skara/ci/Test.java | 46 + settings.gradle | 2 + test/build.gradle | 1 + 39 files changed, 3356 insertions(+) create mode 100644 bots/tester/build.gradle create mode 100644 bots/tester/src/main/java/module-info.java create mode 100644 bots/tester/src/main/java/org/openjdk/skara/bots/tester/Stage.java create mode 100644 bots/tester/src/main/java/org/openjdk/skara/bots/tester/State.java create mode 100644 bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBot.java create mode 100644 bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBotFactory.java create mode 100644 bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestWorkItem.java create mode 100644 bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryContinuousIntegration.java create mode 100644 bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHost.java create mode 100644 bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHostedRepository.java create mode 100644 bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryJob.java create mode 100644 bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryPullRequest.java create mode 100644 bots/tester/src/test/java/org/openjdk/skara/bots/tester/StateTests.java create mode 100644 bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestBotTests.java create mode 100644 bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestWorkItemTests.java create mode 100644 ci/build.gradle create mode 100644 ci/src/main/java/module-info.java create mode 100644 ci/src/main/java/org/openjdk/skara/ci/Build.java create mode 100644 ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegration.java create mode 100644 ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegrationFactory.java create mode 100644 ci/src/main/java/org/openjdk/skara/ci/Job.java create mode 100644 ci/src/main/java/org/openjdk/skara/ci/Test.java diff --git a/bot/build.gradle b/bot/build.gradle index 46bbe201d..5e5c54b8d 100644 --- a/bot/build.gradle +++ b/bot/build.gradle @@ -30,6 +30,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':host') implementation project(':network') implementation project(':issuetracker') diff --git a/bot/src/main/java/module-info.java b/bot/src/main/java/module-info.java index 373540905..92b775627 100644 --- a/bot/src/main/java/module-info.java +++ b/bot/src/main/java/module-info.java @@ -21,6 +21,7 @@ * questions. */ module org.openjdk.skara.bot { + requires transitive org.openjdk.skara.ci; requires transitive org.openjdk.skara.host; requires transitive org.openjdk.skara.issuetracker; requires transitive org.openjdk.skara.forge; diff --git a/bot/src/main/java/org/openjdk/skara/bot/BotConfiguration.java b/bot/src/main/java/org/openjdk/skara/bot/BotConfiguration.java index 448c1fc7d..25a2aaaa0 100644 --- a/bot/src/main/java/org/openjdk/skara/bot/BotConfiguration.java +++ b/bot/src/main/java/org/openjdk/skara/bot/BotConfiguration.java @@ -22,6 +22,7 @@ */ package org.openjdk.skara.bot; +import org.openjdk.skara.ci.ContinuousIntegration; import org.openjdk.skara.forge.HostedRepository; import org.openjdk.skara.issuetracker.IssueProject; import org.openjdk.skara.json.JSONObject; @@ -49,6 +50,13 @@ public interface BotConfiguration { */ IssueProject issueProject(String name); + /** + * Configuration-specific name mapped to a ContinuousIntegration. + * @param name + * @return + */ + ContinuousIntegration continuousIntegration(String name); + /** * Retrieves the ref name that optionally follows the configuration-specific repository name. * If not configured, returns the name of the VCS default branch. diff --git a/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java b/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java index 51b84aabc..1ea73f919 100644 --- a/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java +++ b/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java @@ -22,6 +22,7 @@ */ package org.openjdk.skara.bot; +import org.openjdk.skara.ci.ContinuousIntegration; import org.openjdk.skara.forge.*; import org.openjdk.skara.host.Credential; import org.openjdk.skara.issuetracker.*; @@ -42,6 +43,7 @@ public class BotRunnerConfiguration { private final JSONObject config; private final Map repositoryHosts; private final Map issueHosts; + private final Map continuousIntegrations; private final Map repositories; private BotRunnerConfiguration(JSONObject config, Path cwd) throws ConfigurationError { @@ -50,6 +52,7 @@ private BotRunnerConfiguration(JSONObject config, Path cwd) throws Configuration repositoryHosts = parseRepositoryHosts(config, cwd); issueHosts = parseIssueHosts(config, cwd); + continuousIntegrations = parseContinuousIntegrations(config, cwd); repositories = parseRepositories(config); } @@ -118,6 +121,26 @@ private Map parseIssueHosts(JSONObject config, Path cwd) t return ret; } + private Map parseContinuousIntegrations(JSONObject config, Path cwd) throws ConfigurationError { + Map ret = new HashMap<>(); + + if (!config.contains("ci")) { + return ret; + } + + for (var entry : config.get("ci").fields()) { + var url = entry.value().get("url").asString(); + var ci = ContinuousIntegration.from(URI.create(url), entry.value().asObject()); + if (ci.isPresent()) { + ret.put(entry.name(), ci.get()); + } else { + throw new ConfigurationError("No continuous integration named with url: " + url); + } + } + + return ret; + } + private Map parseRepositories(JSONObject config) throws ConfigurationError { Map ret = new HashMap<>(); @@ -233,6 +256,14 @@ public IssueProject issueProject(String name) { } } + @Override + public ContinuousIntegration continuousIntegration(String name) { + if (continuousIntegrations.containsKey(name)) { + return continuousIntegrations.get(name); + } + throw new RuntimeException("Couldn't find continuous integration with name: " + name); + } + @Override public String repositoryRef(String name) { try { diff --git a/bots/bridgekeeper/build.gradle b/bots/bridgekeeper/build.gradle index a2182ecb6..f70723e89 100644 --- a/bots/bridgekeeper/build.gradle +++ b/bots/bridgekeeper/build.gradle @@ -32,6 +32,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':host') implementation project(':forge') implementation project(':issuetracker') diff --git a/bots/cli/build.gradle b/bots/cli/build.gradle index 6c2c3ee1b..9f63dd050 100644 --- a/bots/cli/build.gradle +++ b/bots/cli/build.gradle @@ -44,8 +44,10 @@ dependencies { implementation project(':bots:mlbridge') implementation project(':bots:mirror') implementation project(':bots:topological') + implementation project(':bots:tester') implementation project(':bots:submit') implementation project(':bots:forward') + implementation project(':ci') implementation project(':vcs') implementation project(':jcheck') implementation project(':host') @@ -74,6 +76,7 @@ images { 'org.openjdk.skara.bots.mlbridge', 'org.openjdk.skara.bots.mirror', 'org.openjdk.skara.bots.submit', + 'org.openjdk.skara.bots.tester', 'org.openjdk.skara.bots.topological', 'org.openjdk.skara.bots.forward'] launchers = ['skara-bots': 'org.openjdk.skara.bots.cli/org.openjdk.skara.bots.cli.BotLauncher'] diff --git a/bots/forward/build.gradle b/bots/forward/build.gradle index 397233bdd..ecf66a510 100644 --- a/bots/forward/build.gradle +++ b/bots/forward/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':host') implementation project(':bot') implementation project(':forge') diff --git a/bots/hgbridge/build.gradle b/bots/hgbridge/build.gradle index 95d99a932..84161605a 100644 --- a/bots/hgbridge/build.gradle +++ b/bots/hgbridge/build.gradle @@ -32,6 +32,7 @@ module { dependencies { implementation project(':bot') + implementation project(':ci') implementation project(':vcs') implementation project(':host') implementation project(':forge') diff --git a/bots/merge/build.gradle b/bots/merge/build.gradle index cfe2030d9..0871ad692 100644 --- a/bots/merge/build.gradle +++ b/bots/merge/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':host') implementation project(':forge') implementation project(':issuetracker') diff --git a/bots/mirror/build.gradle b/bots/mirror/build.gradle index 5341fd05b..631ef2a4e 100644 --- a/bots/mirror/build.gradle +++ b/bots/mirror/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':host') implementation project(':forge') implementation project(':issuetracker') diff --git a/bots/mlbridge/build.gradle b/bots/mlbridge/build.gradle index 059cf2369..b49668a7f 100644 --- a/bots/mlbridge/build.gradle +++ b/bots/mlbridge/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':bot') implementation project(':mailinglist') implementation project(':host') diff --git a/bots/notify/build.gradle b/bots/notify/build.gradle index 7c8483b3c..adebeab41 100644 --- a/bots/notify/build.gradle +++ b/bots/notify/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':host') implementation project(':network') implementation project(':bot') diff --git a/bots/pr/build.gradle b/bots/pr/build.gradle index 3f5f654ae..5b6bb071f 100644 --- a/bots/pr/build.gradle +++ b/bots/pr/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':bot') implementation project(':forge') implementation project(':issuetracker') diff --git a/bots/submit/build.gradle b/bots/submit/build.gradle index 65c22f42f..03896f180 100644 --- a/bots/submit/build.gradle +++ b/bots/submit/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':bot') implementation project(':host') implementation project(':forge') diff --git a/bots/tester/build.gradle b/bots/tester/build.gradle new file mode 100644 index 000000000..ff27e6384 --- /dev/null +++ b/bots/tester/build.gradle @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +module { + name = 'org.openjdk.skara.bots.tester' + test { + requires 'org.junit.jupiter.api' + requires 'org.openjdk.skara.test' + requires 'org.openjdk.skara.host' + opens 'org.openjdk.skara.bots.tester' to 'org.junit.platform.commons' + } +} + +dependencies { + implementation project(':bot') + implementation project(':ci') + implementation project(':census') + implementation project(':forge') + implementation project(':host') + implementation project(':issuetracker') + implementation project(':json') + implementation project(':vcs') + + testImplementation project(':test') +} diff --git a/bots/tester/src/main/java/module-info.java b/bots/tester/src/main/java/module-info.java new file mode 100644 index 000000000..bbe26d952 --- /dev/null +++ b/bots/tester/src/main/java/module-info.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +module org.openjdk.skara.bots.tester { + requires org.openjdk.skara.bot; + requires org.openjdk.skara.vcs; + requires org.openjdk.skara.ci; + + requires java.logging; + + provides org.openjdk.skara.bot.BotFactory with org.openjdk.skara.bots.tester.TestBotFactory; +} diff --git a/bots/tester/src/main/java/org/openjdk/skara/bots/tester/Stage.java b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/Stage.java new file mode 100644 index 000000000..ba2ab8712 --- /dev/null +++ b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/Stage.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +enum Stage { + NA, + ERROR, + REQUESTED, + PENDING, + APPROVED, + STARTED, + CANCELLED, + FINISHED +} diff --git a/bots/tester/src/main/java/org/openjdk/skara/bots/tester/State.java b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/State.java new file mode 100644 index 000000000..e09a81689 --- /dev/null +++ b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/State.java @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.forge.PullRequest; +import org.openjdk.skara.issuetracker.Comment; + +class State { + private final Stage stage; + private final Comment requested; + private final Comment pending; + private final Comment approval; + private final Comment started; + private final Comment cancelled; + private final Comment finished; + + private State(Stage stage, Comment requested, + Comment pending, + Comment approval, + Comment started, + Comment cancelled, + Comment finished) { + this.stage = stage; + this.requested = requested; + this.pending = pending; + this.approval = approval; + this.started = started; + this.cancelled = cancelled; + this.finished = finished; + } + + Stage stage() { + return stage; + } + + Comment requested() { + return requested; + } + + Comment pending() { + return pending; + } + + Comment approval() { + return approval; + } + + Comment started() { + return started; + } + + Comment cancelled() { + return cancelled; + } + + Comment finished() { + return finished; + } + + static State from(PullRequest pr, String approverGroupId) { + Comment requested = null; + Comment pending = null; + Comment approval = null; + Comment started = null; + Comment cancelled = null; + Comment error = null; + Comment finished = null; + + var isApproved = false; + + var host = pr.repository().forge(); + var comments = pr.comments(); + var start = -1; + for (var i = comments.size() - 1; i >=0; i--) { + var comment = comments.get(i); + var lines = comment.body().split("\n"); + if (lines.length == 1 && + lines[0].startsWith("/test") && + !lines[0].startsWith("/test approve") && + !lines[0].startsWith("/test cancel")) { + requested = comment; + start = i; + break; + } + } + + if (requested != null) { + var applicable = comments.subList(start, comments.size()); + for (var comment : applicable) { + var body = comment.body(); + var author = comment.author(); + if (author.equals(host.currentUser())) { + var lines = body.split("\n"); + switch (lines[0]) { + case "": + pending = comment; + break; + case "": + started = comment; + break; + case "": + error = comment; + break; + case "": + finished = comment; + break; + } + } else if (body.equals("/test approve")) { + approval = comment; + if (host.isMemberOf(approverGroupId, author)) { + isApproved = true; + } + } else if (body.equals("/test cancel")) { + if (comment.author().equals(requested.author())) { + cancelled = comment; + } + } else if (body.startsWith("/test")) { + if (host.isMemberOf(approverGroupId, author)) { + isApproved = true; + } + } + } + } + + Stage stage = null; + if (error != null) { + stage = Stage.ERROR; + } else if (cancelled != null) { + stage = Stage.CANCELLED; + } else if (finished != null) { + stage = Stage.FINISHED; + } else if (started != null) { + stage = Stage.STARTED; + } else if (requested != null && isApproved) { + stage = Stage.APPROVED; + } else if (requested != null && pending != null) { + stage = Stage.PENDING; + } else if (requested != null) { + stage = Stage.REQUESTED; + } else { + stage = Stage.NA; + } + + return new State(stage, requested, pending, approval, started, cancelled, finished); + } +} diff --git a/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBot.java b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBot.java new file mode 100644 index 000000000..440b93a3d --- /dev/null +++ b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBot.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.ci.ContinuousIntegration; +import org.openjdk.skara.ci.Job; +import org.openjdk.skara.bot.*; +import org.openjdk.skara.forge.*; + +import java.io.*; +import java.nio.file.*; +import java.util.*; +import java.util.stream.Collectors; + +public class TestBot implements Bot { + private final ContinuousIntegration ci; + private final String approversGroupId; + private final List availableJobs; + private final List defaultJobs; + private final String name; + private final Path storage; + private final HostedRepository repo; + private final PullRequestUpdateCache cache; + private final Set seen; + + TestBot(ContinuousIntegration ci, + String approversGroupId, + List availableJobs, + List defaultJobs, + String name, + Path storage, + HostedRepository repo) { + this.ci = ci; + this.approversGroupId = approversGroupId; + this.availableJobs = availableJobs; + this.defaultJobs = defaultJobs; + this.name = name; + this.storage = storage; + this.repo = repo; + this.cache = new PullRequestUpdateCache(); + this.seen = new HashSet<>(); + } + + @Override + public List getPeriodicItems() { + var ret = new ArrayList(); + + var host = repo.webUrl().getHost(); + var repoId = Long.toString(repo.id()); + for (var pr : repo.pullRequests()) { + if (cache.needsUpdate(pr)) { + ret.add(new TestWorkItem(ci, + approversGroupId, + availableJobs, + defaultJobs, + name, + storage, + pr)); + } else { + // is there a job running for this PR? + var colon = "%3A"; + var asterisk = "%2A"; + var id = host + "-" + repoId + "-"+ pr.id() + "-" + asterisk; + try { + var jobs = ci.query("id" + colon + id); + if (!jobs.isEmpty()) { + if (jobs.stream().anyMatch(j -> j.isRunning() || !seen.contains(j.id()))) { + ret.add(new TestWorkItem(ci, + approversGroupId, + availableJobs, + defaultJobs, + name, + storage, + pr)); + } + seen.addAll(jobs.stream().map(Job::id).collect(Collectors.toList())); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + return ret; + } +} diff --git a/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBotFactory.java b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBotFactory.java new file mode 100644 index 000000000..9ce76b6eb --- /dev/null +++ b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBotFactory.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.bot.*; +import org.openjdk.skara.json.*; + +import org.openjdk.skara.ci.ContinuousIntegration; + +import java.io.*; +import java.nio.file.*; +import java.util.*; +import java.util.stream.Collectors; +import java.net.URI; + +public class TestBotFactory implements BotFactory { + @Override + public String name() { + return "test"; + } + + @Override + public List create(BotConfiguration configuration) { + var storage = configuration.storageFolder(); + try { + Files.createDirectories(storage); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + var ret = new ArrayList(); + var specific = configuration.specific(); + + var approvers = specific.get("approvers").asString(); + var availableJobs = specific.get("availableJobs").stream().map(JSONValue::asString).collect(Collectors.toList()); + var defaultJobs = specific.get("defaultJobs").stream().map(JSONValue::asString).collect(Collectors.toList()); + var name = specific.get("name").asString(); + var ci = configuration.continuousIntegration(specific.get("ci").asString()); + for (var repo : specific.get("repositories").asArray()) { + var hostedRepo = configuration.repository(repo.asString()); + ret.add(new TestBot(ci, approvers, availableJobs, defaultJobs, name, storage, hostedRepo)); + } + + return ret; + } +} diff --git a/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestWorkItem.java b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestWorkItem.java new file mode 100644 index 000000000..af29dc586 --- /dev/null +++ b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestWorkItem.java @@ -0,0 +1,443 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.bot.*; +import org.openjdk.skara.ci.*; +import org.openjdk.skara.forge.*; +import org.openjdk.skara.vcs.*; + +import java.io.*; +import java.net.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.time.ZonedDateTime; +import java.util.*; +import java.util.logging.Logger; +import java.util.stream.*; + +public class TestWorkItem implements WorkItem { + private final Logger log = Logger.getLogger("org.openjdk.skara.bots");; + private final ContinuousIntegration ci; + private final String approversGroupId; + private final List availableJobs; + private final List defaultJobs; + private final String name; + private final Path storage; + private final HostedRepository repository; + private final PullRequest pr; + + TestWorkItem(ContinuousIntegration ci, String approversGroupId, List availableJobs, + List defaultJobs, String name, Path storage, PullRequest pr) { + this.ci = ci; + this.approversGroupId = approversGroupId; + this.availableJobs = availableJobs; + this.defaultJobs = defaultJobs; + this.name = name; + this.storage = storage; + this.repository = pr.repository(); + this.pr = pr; + } + + @Override + public boolean concurrentWith(WorkItem other) { + if (!(other instanceof TestWorkItem)) { + return true; + } + var o = (TestWorkItem) other; + if (!repository.url().equals(o.repository.url())) { + return true; + } + return !pr.id().equals(o.pr.id()); + } + + + private String jobId(State state) { + var host = repository.webUrl().getHost(); + return host + "-" + + Long.toString(repository.id()) + "-"+ + pr.id() + "-" + + state.requested().id(); + } + + + private String osDisplayName(Build.OperatingSystem os) { + switch (os) { + case WINDOWS: + return "Windows"; + case MACOS: + return "macOS"; + case LINUX: + return "Linux"; + case SOLARIS: + return "Solaris"; + case AIX: + return "AIX"; + case FREEBSD: + return "FreeBSD"; + case OPENBSD: + return "OpenBSD"; + case NETBSD: + return "NetBSD"; + case HPUX: + return "HP-UX"; + case HAIKU: + return "Haiku"; + default: + throw new IllegalArgumentException("Unknown operating system: " + os.toString()); + } + } + + private String cpuDisplayName(Build.CPU cpu) { + switch (cpu) { + case X86: + return "x86"; + case X64: + return "x64"; + case SPARCV9: + return "SPARC V9"; + case AARCH64: + return "AArch64"; + case AARCH32: + return "AArch32"; + case PPCLE32: + return "PPC LE 32"; + case PPCLE64: + return "PPC LE 64"; + default: + throw new IllegalArgumentException("Unknown cpu: " + cpu.toString()); + } + } + + private String debugLevelDisplayName(Build.DebugLevel level) { + switch (level) { + case RELEASE: + return "release"; + case FASTDEBUG: + return "fastdebug"; + case SLOWDEBUG: + return "slowdebug"; + default: + throw new IllegalArgumentException("Unknown debug level: " + level.toString()); + } + } + + private void appendIdSection(StringBuilder summary, Job job) { + summary.append("## Id"); + summary.append("\n"); + + summary.append("`"); + summary.append(job.id()); + summary.append("`"); + summary.append("\n"); + } + + private void appendBuildsSection(StringBuilder summary, Job job) { + var perOSandArch = new HashMap>(); + for (var build : job.builds()) { + var osAndArch = osDisplayName(build.os()) + " " + cpuDisplayName(build.cpu()); + var debugLevel = debugLevelDisplayName(build.debugLevel()); + if (!perOSandArch.containsKey(osAndArch)) { + perOSandArch.put(osAndArch, new ArrayList()); + } + perOSandArch.get(osAndArch).add(debugLevel); + } + + summary.append("## Builds"); + summary.append("\n"); + + for (var key : perOSandArch.keySet()) { + summary.append("- "); + summary.append(key); + summary.append(" ("); + summary.append(String.join(",", perOSandArch.get(key))); + summary.append(")"); + summary.append("\n"); + } + } + + private void appendTestsSection(StringBuilder summary, Job job) { + summary.append("## Tests"); + summary.append("\n"); + + for (var test : job.tests()) { + summary.append("- "); + summary.append(test.name()); + summary.append("\n"); + } + } + + private void appendStatusSection(StringBuilder summary, Job job) { + var s = job.status(); + summary.append("## Status"); + summary.append("\n"); + + var numCompleted = s.numCompleted(); + summary.append(Integer.toString(numCompleted)); + summary.append(numCompleted == 1 ? " job " : " jobs "); + summary.append("completed, "); + + var numRunning = s.numRunning(); + summary.append(Integer.toString(numRunning)); + summary.append(numRunning == 1 ? " job " : " jobs "); + summary.append("running, "); + + var numNotStarted = s.numNotStarted(); + summary.append(Integer.toString(numNotStarted)); + summary.append(numNotStarted == 1 ? " job " : " jobs "); + summary.append("not yet started"); + summary.append("\n"); + } + + private void appendResultSection(StringBuilder summary, Job job) { + var r = job.result(); + summary.append("## Result"); + summary.append("\n"); + + var numPassed = r.numPassed(); + summary.append(Integer.toString(numPassed)); + summary.append(numPassed == 1 ? " job " : " jobs "); + summary.append("passed, "); + + var numFailed = r.numFailed(); + summary.append(Integer.toString(numFailed)); + summary.append(numFailed == 1 ? " job " : " jobs "); + summary.append("with failures, "); + + var numSkipped = r.numSkipped(); + summary.append(Integer.toString(numSkipped)); + summary.append(numSkipped == 1 ? " job " : " jobs "); + summary.append("not run"); + summary.append("\n"); + } + + private String display(Job job) { + var sb = new StringBuilder(); + appendIdSection(sb, job); + sb.append("\n"); + appendBuildsSection(sb, job); + sb.append("\n"); + appendTestsSection(sb, job); + sb.append("\n"); + appendStatusSection(sb, job); + sb.append("\n"); + if (job.isCompleted()) { + appendResultSection(sb, job); + } + return sb.toString(); + } + + @Override + public void run(Path scratchPath) { + var state = State.from(pr, approversGroupId); + var stage = state.stage(); + if (stage == Stage.NA || stage == Stage.ERROR || stage == Stage.PENDING || stage == Stage.FINISHED) { + // nothing to do + return; + } + + if (stage == Stage.STARTED) { + if (state.started() != null) { + var lines = state.started().body().split("\n"); + var jobId = lines[1].replace("", ""); + var hash = lines[2].replace("", ""); + + try { + var job = ci.job(jobId); + var checks = pr.checks(new Hash(hash)); + if (checks.containsKey(name)) { + var check = checks.get(name); + if (check.status() == CheckStatus.IN_PROGRESS) { + var builder = CheckBuilder.from(check); + if (job.isCompleted()) { + var success = job.result().numFailed() == 0 && + job.result().numSkipped() == 0; + builder = builder.complete(success); + var requestor = state.requested().author().userName(); + var commentLines = List.of( + "", + "", + "", + "@" + requestor + " your test job with id " + jobId + " for commits up until " + hash.substring(0, 8) + " has finished." + ); + pr.addComment(String.join("\n", commentLines)); + } + builder = builder.summary(display(job)); + pr.updateCheck(builder.build()); + } + } else { + log.warning("Could not find check for job with " + jobId + " for hash " + hash + " for PR " + pr.webUrl()); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + log.warning("No 'started' comment present for PR " + pr.webUrl()); + } + } else if (stage == stage.CANCELLED) { + if (state.started() != null) { + var lines = state.started().body().split("\n"); + var jobId = lines[1].replace("", ""); + var hash = lines[2].replace("", ""); + + try { + ci.cancel(jobId); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + var checks = pr.checks(new Hash(hash)); + if (checks.containsKey(name)) { + var check = checks.get(name); + if (check.status() != CheckStatus.CANCELLED) { + var builder = CheckBuilder.from(check); + var newCheck = builder.cancel() + .build(); + pr.updateCheck(newCheck); + } + } else { + log.warning("Could not find check for job with " + jobId + " for hash " + hash + " for PR " + pr.webUrl()); + } + } + } else if (stage == Stage.REQUESTED) { + var requestedJobs = state.requested().body().substring("/test".length()); + if (requestedJobs.trim().isEmpty()) { + requestedJobs = String.join(",", defaultJobs); + } + var trimmedJobs = Stream.of(requestedJobs.split(",")).map(String::trim).collect(Collectors.toList()); + var nonExistingJobs = trimmedJobs.stream().filter(s -> !availableJobs.contains(s)) + .collect(Collectors.toList()); + if (!nonExistingJobs.isEmpty()) { + var wording = nonExistingJobs.size() == 1 ? "group " : "groups "; + var lines = List.of( + "", + "@" + state.requested().author().userName() + " the test " + wording + String.join(",", nonExistingJobs) + " does not exist" + ); + pr.addComment(String.join("\n", lines)); + } else { + var head = pr.headHash(); + var lines = List.of( + "", + "", + "", + "@" + state.requested().author().userName() + " you need to get approval to run the tests in " + + String.join(",", trimmedJobs) + " for commits up until " + head.abbreviate() + ); + pr.addComment(String.join("\n", lines)); + } + } else if (stage == Stage.APPROVED) { + Hash head = null; + List jobs = null; + + if (state.pending() != null) { + var comment = state.pending(); + var body = comment.body().split("\n"); + + head = new Hash(body[1].replace("", "")); + var requestedJobs = body[2].replace("", ""); + jobs = Arrays.asList(requestedJobs.split(",")); + } else { + var comment = state.requested(); + var body = comment.body().split("\n"); + + head = pr.headHash(); + var requestedJobs = state.requested().body().substring("/test".length()); + if (requestedJobs.trim().isEmpty()) { + requestedJobs = String.join(",", defaultJobs); + } + var trimmedJobs = Stream.of(requestedJobs.split(",")).map(String::trim).collect(Collectors.toList()); + var nonExistingJobs = trimmedJobs.stream().filter(s -> !availableJobs.contains(s)) + .collect(Collectors.toList()); + if (!nonExistingJobs.isEmpty()) { + var wording = nonExistingJobs.size() == 1 ? "group " : "groups "; + var lines = List.of( + "", + "@" + state.requested().author().userName() + " the test " + wording + String.join(",", nonExistingJobs) + " does not exist" + ); + pr.addComment(String.join("\n", lines)); + return; + } + + jobs = trimmedJobs; + } + var jobId = jobId(state); + + Job job = null; + Hash fetchHead = null; + try { + var sanitizedUrl = URLEncoder.encode(repository.webUrl().toString(), StandardCharsets.UTF_8); + var localRepoDir = storage.resolve("mach5-bot") + .resolve(sanitizedUrl) + .resolve(pr.id()); + var host = repository.webUrl().getHost(); + Repository localRepo = null; + if (!Files.exists(localRepoDir)) { + log.info("Cloning " + repository.name()); + Files.createDirectories(localRepoDir); + var url = repository.webUrl().toString(); + if (!url.endsWith(".git")) { + url += ".git"; + } + localRepo = Repository.clone(URI.create(url), localRepoDir); + } else { + log.info("Found existing scratch directory for " + repository.name()); + localRepo = Repository.get(localRepoDir).orElseThrow(() -> { + return new RuntimeException("Repository in " + localRepoDir + " has vanished"); + }); + } + fetchHead = localRepo.fetch(repository.url(), pr.targetRef()); + localRepo.checkout(fetchHead, true); + job = ci.submit(localRepoDir, jobs, jobId); + } catch (IOException e) { + var lines = List.of( + "", + "Could not create test job" + ); + pr.addComment(String.join("\n", lines)); + + throw new UncheckedIOException(e); + } + + var check = CheckBuilder.create(name, fetchHead) + .title("Summary") + .summary(display(job)) + .metadata(jobId) + .build(); + pr.createCheck(check); + + var lines = List.of( + "", + "", + "", + "A test job has been started with id: " + jobId + ); + pr.addComment(String.join("\n", lines)); + } else { + throw new RuntimeException("Unexpected state " + state); + } + } + + @Override + public String toString() { + return "TestWorkItem@" + pr.repository().name() + "#" + pr.id(); + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryContinuousIntegration.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryContinuousIntegration.java new file mode 100644 index 000000000..280b049db --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryContinuousIntegration.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.ci.ContinuousIntegration; +import org.openjdk.skara.ci.Job; +import org.openjdk.skara.host.HostUser; + +import java.io.*; +import java.nio.file.*; +import java.util.*; + +class InMemoryContinuousIntegration implements ContinuousIntegration { + static class Submission { + Path source; + List jobs; + String id; + + Submission(Path source, List jobs, String id) { + this.source = source; + this.jobs = jobs; + this.id = id; + } + } + + List submissions = new ArrayList(); + List cancelled = new ArrayList(); + Map jobs = new HashMap<>(); + boolean throwOnSubmit = false; + boolean isValid = true; + Map users = new HashMap<>(); + HostUser currentUser = null; + Map> groups = new HashMap<>(); + + @Override + public boolean isValid() { + return isValid; + } + + @Override + public HostUser user(String username) { + return users.get(username); + } + + @Override + public HostUser currentUser() { + return currentUser; + } + + @Override + public boolean isMemberOf(String groupId, HostUser user) { + var group = groups.get(groupId); + return group != null && group.contains(user); + } + + @Override + public Job submit(Path source, List jobs, String id) throws IOException { + if (throwOnSubmit) { + throw new IOException("Something went wrong"); + } + submissions.add(new Submission(source, jobs, id)); + return job(id); + } + + @Override + public Job job(String id) throws IOException { + return jobs.get(id); + } + + @Override + public void cancel(String id) throws IOException { + cancelled.add(id); + } + + @Override + public List query(String query) throws IOException { + return List.of(); + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHost.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHost.java new file mode 100644 index 000000000..ae7f2008b --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHost.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.forge.*; +import org.openjdk.skara.host.*; + +import java.util.*; + +class InMemoryHost implements Forge { + HostUser currentUserDetails = new HostUser(0, "openjdk", "openjdk [bot]"); + Map> groups; + + @Override + public boolean isValid() { + return false; + } + + @Override + public HostedRepository repository(String name) { + return null; + } + + @Override + public HostUser user(String username) { + return null; + } + + @Override + public HostUser currentUser() { + return currentUserDetails; + } + + @Override + public boolean supportsReviewBody() { + return false; + } + + @Override + public boolean isMemberOf(String groupId, HostUser user) { + return groups.get(groupId).contains(user); + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHostedRepository.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHostedRepository.java new file mode 100644 index 000000000..98e6f9dec --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHostedRepository.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.forge.*; +import org.openjdk.skara.json.JSONValue; +import org.openjdk.skara.vcs.*; + +import java.net.URI; +import java.util.*; + +class InMemoryHostedRepository implements HostedRepository { + Forge host; + URI webUrl; + URI url; + long id; + + @Override + public Forge forge() { + return host; + } + + @Override + public PullRequest createPullRequest(HostedRepository target, + String targetRef, + String sourceRef, + String title, + List body, + boolean draft) { + return null; + } + + @Override + public PullRequest pullRequest(String id) { + return null; + } + + @Override + public List pullRequests() { + return null; + } + + @Override + public Optional parsePullRequestUrl(String url) { + return null; + } + + @Override + public String name() { + return null; + } + + @Override + public Optional parent() { + return null; + } + + @Override + public URI url() { + return url; + } + + @Override + public URI webUrl() { + return webUrl; + } + + @Override + public URI webUrl(Hash hash) { + return null; + } + + @Override + public VCS repositoryType() { + return null; + } + + @Override + public String fileContents(String filename, String ref) { + return null; + } + + @Override + public String namespace() { + return null; + } + + @Override + public Optional parseWebHook(JSONValue body) { + return null; + } + + @Override + public HostedRepository fork() { + return null; + } + + @Override + public long id() { + return id; + } + + @Override + public Hash branchHash(String ref) { + return null; + } + + @Override + public List findPullRequestsWithComment(String author, String body) { + return null; + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryJob.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryJob.java new file mode 100644 index 000000000..c9ecdd262 --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryJob.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.ci.*; + +import java.util.List; +import java.util.ArrayList; + +class InMemoryJob implements Job { + String id = ""; + List builds = new ArrayList<>(); + List tests = new ArrayList<>(); + Job.Status status; + Job.Result result; + Job.State state; + + @Override + public String id() { + return id; + } + + @Override + public List builds() { + return builds; + } + + @Override + public List tests() { + return tests; + } + + @Override + public Job.Status status() { + return status; + } + + @Override + public Job.Result result() { + return result; + } + + @Override + public Job.State state() { + return state; + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryPullRequest.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryPullRequest.java new file mode 100644 index 000000000..a19b41034 --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryPullRequest.java @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.forge.*; +import org.openjdk.skara.host.*; +import org.openjdk.skara.issuetracker.*; +import org.openjdk.skara.vcs.*; + +import java.util.*; +import java.time.*; +import java.net.*; + +class InMemoryPullRequest implements PullRequest { + List comments = new ArrayList(); + List reviews = new ArrayList(); + HostUser author; + HostedRepository repository; + Hash headHash; + String id; + String targetRef; + Map> checks = new HashMap<>(); + + @Override + public HostedRepository repository() { + return repository; + } + + @Override + public String id() { + return id; + } + + @Override + public HostUser author() { + return author; + } + + @Override + public List reviews() { + return reviews; + } + + @Override + public void addReview(Review.Verdict verdict, String body) { + } + + @Override + public ReviewComment addReviewComment(Hash base, Hash hash, String path, int line, String body) { + return null; + } + + @Override + public ReviewComment addReviewCommentReply(ReviewComment parent, String body) { + return null; + } + + @Override + public List reviewComments() { + return null; + } + + @Override + public Hash headHash() { + return headHash; + } + + @Override + public String sourceRef() { + return null; + } + + @Override + public String targetRef() { + return targetRef; + } + + @Override + public Hash targetHash() { + return null; + } + + @Override + public String title() { + return null; + } + + @Override + public String body() { + return null; + } + + @Override + public void setBody(String body) { + } + + @Override + public List comments() { + return comments; + } + void setComments(List comments) { + this.comments = comments; + } + + @Override + public Comment addComment(String body) { + var user = repository().forge().currentUser(); + var now = ZonedDateTime.now(); + var id = comments.size(); + var comment = new Comment(Integer.toString(id), body, user, now, now); + comments.add(comment); + return comment; + } + + @Override + public Comment updateComment(String id, String body) { + var index = Integer.parseInt(id); + var old = comments.get(index); + + var now = ZonedDateTime.now(); + var newComment = new Comment(id, body, old.author(), old.createdAt(), now); + comments.set(index, newComment); + return newComment; + } + + @Override + public ZonedDateTime createdAt() { + return null; + } + + @Override + public ZonedDateTime updatedAt() { + return null; + } + + @Override + public Map checks(Hash hash) { + return checks.get(hash.hex()); + } + + @Override + public void createCheck(Check check) { + if (!checks.containsKey(check.hash().hex())) { + checks.put(check.hash().hex(), new HashMap<>()); + } + checks.get(check.hash().hex()).put(check.name(), check); + } + + @Override + public void updateCheck(Check check) { + if (checks.containsKey(check.hash().hex())) { + checks.get(check.hash().hex()).put(check.name(), check); + } + } + + @Override + public URI changeUrl() { + return null; + } + + @Override + public URI changeUrl(Hash base) { + return null; + } + + @Override + public boolean isDraft() { + return false; + } + + @Override + public void setState(State state) { + } + + @Override + public void addLabel(String label) { + } + + @Override + public void removeLabel(String label) { + } + + @Override + public List labels() { + return null; + } + + @Override + public URI webUrl() { + return null; + } + + @Override + public List assignees() { + return null; + } + + @Override + public void setAssignees(List assignees) { + } + + @Override + public void setTitle(String title) { + } + + @Override + public IssueProject project() { + return null; + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/StateTests.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/StateTests.java new file mode 100644 index 000000000..2f1d54eb2 --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/StateTests.java @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.issuetracker.Comment; +import org.openjdk.skara.host.HostUser; + +import java.util.*; +import java.time.ZonedDateTime; + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +class StateTests { + @Test + void noCommentsShouldEqualNA() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.comments = List.of(); + + var state = State.from(pr, "0"); + assertEquals(Stage.NA, state.stage()); + assertEquals(null, state.requested()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void testCommentFromNotApprovedUserShouldEqualRequested() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = List.of(comment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of()); + + var state = State.from(pr, approvers); + assertEquals(Stage.REQUESTED, state.stage()); + assertEquals(comment, state.requested()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void testCommentFromApprovedUserShouldEqualApproved() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = List.of(comment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of(duke)); + + var state = State.from(pr, approvers); + assertEquals(Stage.APPROVED, state.stage()); + assertEquals(comment, state.requested()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void testApprovalNeededCommentShouldResultInPending() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + + var pendingBody = List.of( + "", + "", + "@duke you need to get approval to run these tests" + ); + var pendingComment = new Comment("0", String.join("\n", pendingBody), bot, now, now); + pr.comments = List.of(testComment, pendingComment); + host.groups = Map.of("0", Set.of()); + + var state = State.from(pr, "0"); + assertEquals(Stage.PENDING, state.stage()); + assertEquals(testComment, state.requested()); + assertEquals(pendingComment, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void testStartedCommentShouldResultInRunning() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + + var pendingBody = List.of( + "", + "", + "@duke you need to get approval to run these tests" + ); + var pendingComment = new Comment("1", String.join("\n", pendingBody), bot, now, now); + + var member = new HostUser(2, "foo", "Foo Bar"); + var approveComment = new Comment("2", "/test approve", member, now, now); + + var startedBody = List.of( + "", + "", + "A test job has been started with id 0" + ); + var startedComment = new Comment("3", String.join("\n", startedBody), bot, now, now); + + pr.comments = List.of(testComment, pendingComment, approveComment, startedComment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of(member)); + + var state = State.from(pr, approvers); + assertEquals(Stage.STARTED, state.stage()); + assertEquals(testComment, state.requested()); + assertEquals(pendingComment, state.pending()); + assertEquals(startedComment, state.started()); + } + + @Test + void cancelCommentFromAuthorShouldEqualCancelled() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + var cancelComment = new Comment("1", "/test cancel", duke, now, now); + pr.comments = List.of(testComment, cancelComment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of()); + + var state = State.from(pr, approvers); + assertEquals(Stage.CANCELLED, state.stage()); + assertEquals(testComment, state.requested()); + assertEquals(cancelComment, state.cancelled()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void cancelCommentFromAnotherUserShouldHaveNoEffect() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var user = new HostUser(0, "foo", "Foo Bar"); + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + var cancelComment = new Comment("1", "/test cancel", user, now, now); + pr.comments = List.of(testComment, cancelComment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of()); + + var state = State.from(pr, approvers); + assertEquals(Stage.REQUESTED, state.stage()); + assertEquals(testComment, state.requested()); + assertEquals(null, state.cancelled()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void multipleTestCommentsShouldOnlyCareAboutLast() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var test1Comment = new Comment("0", "/test tier1", duke, now, now); + var test2Comment = new Comment("1", "/test tier1,tier2", duke, now, now); + var test3Comment = new Comment("2", "/test tier1,tier2,tier3", duke, now, now); + pr.comments = List.of(test1Comment, test2Comment, test3Comment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of()); + + var state = State.from(pr, approvers); + assertEquals(Stage.REQUESTED, state.stage()); + assertEquals(test3Comment, state.requested()); + assertEquals(null, state.cancelled()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void errorAfterRequestedShouldBeError() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + + var lines = List.of( + "", + "The test tier1 does not exist" + ); + var errorComment = new Comment("2", String.join("\n", lines), bot, now, now); + pr.comments = List.of(testComment, errorComment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of()); + + var state = State.from(pr, approvers); + assertEquals(Stage.ERROR, state.stage()); + assertEquals(testComment, state.requested()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void testFinishedCommentShouldResultInFinished() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + + var pendingBody = List.of( + "", + "", + "@duke you need to get approval to run these tests" + ); + var pendingComment = new Comment("1", String.join("\n", pendingBody), bot, now, now); + + var member = new HostUser(2, "foo", "Foo Bar"); + var approveComment = new Comment("2", "/test approve", member, now, now); + + var startedBody = List.of( + "", + "", + "A test job has been started with id 0" + ); + var startedComment = new Comment("3", String.join("\n", startedBody), bot, now, now); + + var finishedBody = List.of( + "", + "", + "A test job has been started with id 0" + ); + var finishedComment = new Comment("4", String.join("\n", finishedBody), bot, now, now); + + pr.comments = List.of(testComment, pendingComment, approveComment, startedComment, finishedComment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of(member)); + + var state = State.from(pr, approvers); + assertEquals(Stage.FINISHED, state.stage()); + assertEquals(testComment, state.requested()); + assertEquals(pendingComment, state.pending()); + assertEquals(startedComment, state.started()); + assertEquals(finishedComment, state.finished()); + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestBotTests.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestBotTests.java new file mode 100644 index 000000000..d8ca2c4a7 --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestBotTests.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.test.*; + +import java.io.*; +import java.util.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import static org.junit.jupiter.api.Assertions.*; + +class TestBotTests { + @Test + void noTestCommentShouldDoNothing(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tmp = new TemporaryDirectory()) { + var upstreamHostedRepo = credentials.getHostedRepository(); + var personalHostedRepo = credentials.getHostedRepository(); + var pr = personalHostedRepo.createPullRequest(upstreamHostedRepo, + "master", + "master", + "Title", + List.of("body")); + + var comments = pr.comments(); + assertEquals(0, comments.size()); + + var storage = tmp.path().resolve("storage"); + var ci = new InMemoryContinuousIntegration(); + var bot = new TestBot(ci, "0", List.of(), List.of(), "", storage, upstreamHostedRepo); + var runner = new TestBotRunner(); + + runner.runPeriodicItems(bot); + + comments = pr.comments(); + assertEquals(0, comments.size()); + } + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestWorkItemTests.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestWorkItemTests.java new file mode 100644 index 000000000..4f1c0133b --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestWorkItemTests.java @@ -0,0 +1,952 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.forge.CheckStatus; +import org.openjdk.skara.host.*; +import org.openjdk.skara.issuetracker.Comment; +import org.openjdk.skara.vcs.*; +import org.openjdk.skara.test.*; +import org.openjdk.skara.ci.Job; + +import java.io.*; +import java.net.URI; +import java.nio.file.*; +import java.util.*; +import java.time.ZonedDateTime; + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +class TestWorkItemTests { + @Test + void noTestCommentsShouldDoNothing() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.comments = List.of(); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + item.run(scratch); + + var comments = pr.comments(); + assertEquals(0, comments.size()); + } + } + + @Test + void topLevelTestApproveShouldDoNothing() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + var now = ZonedDateTime.now(); + pr.author = duke; + var testApproveComment = new Comment("0", "/test approve", duke, now, now); + pr.comments = List.of(testApproveComment); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + item.run(scratch); + + var comments = pr.comments(); + assertEquals(1, comments.size()); + assertEquals(testApproveComment, comments.get(0)); + } + } + + @Test + void topLevelTestCancelShouldDoNothing() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + var now = ZonedDateTime.now(); + pr.author = duke; + var testApproveComment = new Comment("0", "/test cancel", duke, now, now); + pr.comments = List.of(testApproveComment); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + item.run(scratch); + + var comments = pr.comments(); + assertEquals(1, comments.size()); + assertEquals(testApproveComment, comments.get(0)); + } + } + + @Test + void testCommentWithMadeUpJobShouldBeError() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of("0", Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test foobar", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals(2, lines.length); + assertEquals("", lines[0]); + assertEquals("@duke the test group foobar does not exist", lines[1]); + } + } + + @Test + void testCommentFromUnapprovedUserShouldBePending() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of("0", Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = new Hash("01234567890123456789012345789012345789"); + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test foobar", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + // Non-existing test group should result in error + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals(2, lines.length); + assertEquals("", lines[0]); + assertEquals("@duke the test group foobar does not exist", lines[1]); + + // Trying to test again should be fine + var thirdComment = new Comment("2", "/test tier1", duke, now, now); + pr.comments.add(thirdComment); + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(thirdComment, comments.get(2)); + + var fourthComment = comments.get(3); + assertEquals(bot, fourthComment.author()); + + lines = fourthComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until 01234567", + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(thirdComment, comments.get(2)); + assertEquals(fourthComment, comments.get(3)); + } + } + + @Test + void cancelAtestCommentShouldBeCancel() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of("0", Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = new Hash("01234567890123456789012345789012345789"); + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + var cancelComment = new Comment("1", "/test cancel", duke, now, now); + pr.comments = new ArrayList<>(List.of(testComment, cancelComment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(testComment, comments.get(0)); + assertEquals(cancelComment, comments.get(1)); + } + } + + @Test + void cancellingAPendingTestCommentShouldWork() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of(approvers, Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = new Hash("01234567890123456789012345789012345789"); + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until 01234567", + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + + // Cancelling the test now should be fine + var cancelComment = new Comment("2", "/test cancel", duke, now, now); + pr.comments.add(cancelComment); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(3, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(cancelComment, comments.get(2)); + + // Approving the test should not start a job, it has already been cancelled + var member = new HostUser(3, "foo", "Foo Bar"); + host.groups = Map.of(approvers, Set.of(member)); + var approveComment = new Comment("3", "/test approve", member, now, now); + pr.comments.add(approveComment); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(cancelComment, comments.get(2)); + assertEquals(approveComment, comments.get(3)); + } + } + + @Test + void cancellingApprovedPendingRequestShouldBeCancelled() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of(approvers, Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = new Hash("01234567890123456789012345789012345789"); + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until 01234567", + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + + // Approve the request + var member = new HostUser(2, "foo", "Foo Bar"); + host.groups = Map.of(approvers, Set.of(member)); + var approveComment = new Comment("2", "/test approve", member, now, now); + pr.comments.add(approveComment); + + // Cancelling the request + var cancelComment = new Comment("2", "/test cancel", duke, now, now); + pr.comments.add(cancelComment); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(approveComment, comments.get(2)); + assertEquals(cancelComment, comments.get(3)); + } + } + + @Test + void approvedPendingRequestShouldBeStarted() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var localRepoDir = tmp.path().resolve("repository.git"); + var localRepo = Repository.init(localRepoDir, VCS.GIT); + var readme = localRepoDir.resolve("README"); + Files.writeString(readme, "Hello\n"); + localRepo.add(readme); + var head = localRepo.commit("Add README", "duke", "duke@openjdk.org"); + + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of(approvers, Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + repo.webUrl = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.url = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.id = 1337L; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + pr.id = "17"; + pr.targetRef = "master"; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = head; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until " + head.abbreviate(), + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + + // Approve the request + var member = new HostUser(2, "foo", "Foo Bar"); + host.groups = Map.of(approvers, Set.of(member)); + var approveComment = new Comment("2", "/test approve", member, now, now); + pr.comments.add(approveComment); + + var expectedJobId = "null-1337-17-0"; + var expectedJob = new InMemoryJob(); + expectedJob.status = new Job.Status(0, 1, 7); + ci.jobs.put(expectedJobId, expectedJob); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(approveComment, comments.get(2)); + + var fourthComment = comments.get(3); + lines = fourthComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("A test job has been started with id: " + expectedJobId, lines[3]); + + assertEquals(1, ci.submissions.size()); + var submission = ci.submissions.get(0); + assertTrue(submission.source.startsWith(storage)); + assertEquals(List.of("tier1"), submission.jobs); + assertEquals(expectedJobId, submission.id); + + var checks = pr.checks(pr.headHash()); + assertEquals(1, checks.keySet().size()); + var check = checks.get("test"); + assertEquals("Summary", check.title().get()); + assertTrue(check.summary() + .get() + .contains("0 jobs completed, 1 job running, 7 jobs not yet started")); + } + } + + @Test + void cancellingApprovedPendingRequestShouldBeCancel() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var localRepoDir = tmp.path().resolve("repository.git"); + var localRepo = Repository.init(localRepoDir, VCS.GIT); + var readme = localRepoDir.resolve("README"); + Files.writeString(readme, "Hello\n"); + localRepo.add(readme); + var head = localRepo.commit("Add README", "duke", "duke@openjdk.org"); + + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of(approvers, Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + repo.webUrl = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.url = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.id = 1337L; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + pr.id = "17"; + pr.targetRef = "master"; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = head; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until " + head.abbreviate(), + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + + // Approve the request + var member = new HostUser(2, "foo", "Foo Bar"); + host.groups = Map.of(approvers, Set.of(member)); + var approveComment = new Comment("2", "/test approve", member, now, now); + pr.comments.add(approveComment); + + var expectedJobId = "null-1337-17-0"; + var expectedJob = new InMemoryJob(); + expectedJob.status = new Job.Status(0, 1, 7); + ci.jobs.put(expectedJobId, expectedJob); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(approveComment, comments.get(2)); + + var fourthComment = comments.get(3); + lines = fourthComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("A test job has been started with id: " + expectedJobId, lines[3]); + + assertEquals(1, ci.submissions.size()); + var submission = ci.submissions.get(0); + assertTrue(submission.source.startsWith(storage)); + assertEquals(List.of("tier1"), submission.jobs); + assertEquals(expectedJobId, submission.id); + + var checks = pr.checks(pr.headHash()); + assertEquals(1, checks.keySet().size()); + var check = checks.get("test"); + assertEquals("Summary", check.title().get()); + assertEquals(CheckStatus.IN_PROGRESS, check.status()); + assertTrue(check.summary() + .get() + .contains("## Status\n0 jobs completed, 1 job running, 7 jobs not yet started\n")); + + var cancelComment = new Comment("4", "/test cancel", duke, now, now); + pr.comments.add(cancelComment); + + item.run(scratch); + + checks = pr.checks(pr.headHash()); + assertEquals(1, checks.keySet().size()); + check = checks.get("test"); + assertEquals("Summary", check.title().get()); + assertEquals(CheckStatus.CANCELLED, check.status()); + assertTrue(check.summary() + .get() + .contains("## Status\n0 jobs completed, 1 job running, 7 jobs not yet started\n")); + + assertEquals(expectedJobId, ci.cancelled.get(0)); + } + } + + @Test + void errorWhenCreatingTestJobShouldResultInError() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var localRepoDir = tmp.path().resolve("repository.git"); + var localRepo = Repository.init(localRepoDir, VCS.GIT); + var readme = localRepoDir.resolve("README"); + Files.writeString(readme, "Hello\n"); + localRepo.add(readme); + var head = localRepo.commit("Add README", "duke", "duke@openjdk.org"); + + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of(approvers, Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + repo.webUrl = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.url = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.id = 1337L; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + pr.id = "17"; + pr.targetRef = "master"; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = head; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until " + head.abbreviate(), + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + + // Approve the request + var member = new HostUser(2, "foo", "Foo Bar"); + host.groups = Map.of(approvers, Set.of(member)); + var approveComment = new Comment("2", "/test approve", member, now, now); + pr.comments.add(approveComment); + + ci.throwOnSubmit = true; + assertThrows(UncheckedIOException.class, () -> item.run(scratch)); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(approveComment, comments.get(2)); + + var fifthComment = comments.get(3); + lines = fifthComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("Could not create test job", lines[1]); + } + } + + @Test + void finishedJobShouldResultInFinishedComment() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var localRepoDir = tmp.path().resolve("repository.git"); + var localRepo = Repository.init(localRepoDir, VCS.GIT); + var readme = localRepoDir.resolve("README"); + Files.writeString(readme, "Hello\n"); + localRepo.add(readme); + var head = localRepo.commit("Add README", "duke", "duke@openjdk.org"); + + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of(approvers, Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + repo.webUrl = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.url = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.id = 1337L; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + pr.id = "17"; + pr.targetRef = "master"; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = head; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until " + head.abbreviate(), + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + + // Approve the request + var member = new HostUser(2, "foo", "Foo Bar"); + host.groups = Map.of(approvers, Set.of(member)); + var approveComment = new Comment("2", "/test approve", member, now, now); + pr.comments.add(approveComment); + + var expectedJobId = "null-1337-17-0"; + var expectedJob = new InMemoryJob(); + expectedJob.status = new Job.Status(0, 1, 7); + ci.jobs.put(expectedJobId, expectedJob); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(approveComment, comments.get(2)); + + var fourthComment = comments.get(3); + lines = fourthComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("A test job has been started with id: " + expectedJobId, lines[3]); + + assertEquals(1, ci.submissions.size()); + var submission = ci.submissions.get(0); + assertTrue(submission.source.startsWith(storage)); + assertEquals(List.of("tier1"), submission.jobs); + assertEquals(expectedJobId, submission.id); + + var checks = pr.checks(pr.headHash()); + assertEquals(1, checks.keySet().size()); + var check = checks.get("test"); + assertEquals("Summary", check.title().get()); + assertEquals(CheckStatus.IN_PROGRESS, check.status()); + assertTrue(check.summary() + .get() + .contains("0 jobs completed, 1 job running, 7 jobs not yet started")); + + var job = ci.jobs.get(expectedJobId); + assertNotNull(job); + job.id = "id"; + job.state = Job.State.COMPLETED; + job.status = new Job.Status(8, 0, 0); + job.result = new Job.Result(8, 0, 0); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(5, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(approveComment, comments.get(2)); + assertEquals(fourthComment, comments.get(3)); + + var finishedComment = comments.get(4); + lines = finishedComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke your test job with id " + expectedJobId + " for commits up until " + + head.abbreviate() + " has finished.", lines[3]); + + checks = pr.checks(pr.headHash()); + assertEquals(1, checks.keySet().size()); + check = checks.get("test"); + assertEquals("Summary", check.title().get()); + assertEquals(CheckStatus.SUCCESS, check.status()); + + var summaryLines = check.summary().get().split("\n"); + assertEquals("## Id", summaryLines[0]); + assertEquals("`id`", summaryLines[1]); + assertEquals("", summaryLines[2]); + assertEquals("## Builds", summaryLines[3]); + assertEquals("", summaryLines[4]); + assertEquals("## Tests", summaryLines[5]); + assertEquals("", summaryLines[6]); + assertEquals("## Status", summaryLines[7]); + assertEquals("8 jobs completed, 0 jobs running, 0 jobs not yet started", summaryLines[8]); + assertEquals("", summaryLines[9]); + assertEquals("## Result", summaryLines[10]); + assertEquals("8 jobs passed, 0 jobs with failures, 0 jobs not run", summaryLines[11]); + } + } +} diff --git a/bots/topological/build.gradle b/bots/topological/build.gradle index b7b7e36bc..bff8fb6c7 100644 --- a/bots/topological/build.gradle +++ b/bots/topological/build.gradle @@ -32,6 +32,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':host') implementation project(':forge') implementation project(':issuetracker') diff --git a/ci/build.gradle b/ci/build.gradle new file mode 100644 index 000000000..1c4afd9e2 --- /dev/null +++ b/ci/build.gradle @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +module { + name = 'org.openjdk.skara.ci' + test { + requires 'org.junit.jupiter.api' + requires 'org.openjdk.skara.test' + opens 'org.openjdk.skara.ci' to 'org.junit.platform.commons' + } +} + +dependencies { + implementation project(':host') + implementation project(':json') +} diff --git a/ci/src/main/java/module-info.java b/ci/src/main/java/module-info.java new file mode 100644 index 000000000..3247e493a --- /dev/null +++ b/ci/src/main/java/module-info.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +module org.openjdk.skara.ci { + requires org.openjdk.skara.host; + requires org.openjdk.skara.json; + + uses org.openjdk.skara.ci.ContinuousIntegrationFactory; + exports org.openjdk.skara.ci; +} diff --git a/ci/src/main/java/org/openjdk/skara/ci/Build.java b/ci/src/main/java/org/openjdk/skara/ci/Build.java new file mode 100644 index 000000000..32910c64b --- /dev/null +++ b/ci/src/main/java/org/openjdk/skara/ci/Build.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.ci; + +import java.util.Objects; + +public class Build { + public static enum OperatingSystem { + WINDOWS, + MACOS, + LINUX, + SOLARIS, + AIX, + FREEBSD, + OPENBSD, + NETBSD, + HPUX, + HAIKU + } + + public static enum CPU { + X86, + X64, + SPARCV9, + AARCH64, + AARCH32, + PPCLE32, + PPCLE64 + } + + public static enum DebugLevel { + RELEASE, + FASTDEBUG, + SLOWDEBUG + } + + private final OperatingSystem os; + private final CPU cpu; + private final DebugLevel debugLevel; + + public Build(OperatingSystem os, CPU cpu, DebugLevel debugLevel) { + this.os = os; + this.cpu = cpu; + this.debugLevel = debugLevel; + } + + public OperatingSystem os() { + return os; + } + + public CPU cpu() { + return cpu; + } + + public DebugLevel debugLevel() { + return debugLevel; + } + + @Override + public String toString() { + return os.toString().toLowerCase() + "-" + + cpu.toString().toLowerCase() + "-" + + debugLevel.toString().toLowerCase(); + } + + @Override + public int hashCode() { + return Objects.hash(os, cpu, debugLevel); + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + + if (!(other instanceof Build)) { + return false; + } + + var o = (Build) other; + return Objects.equals(os, o.os) && + Objects.equals(cpu, o.cpu) && + Objects.equals(debugLevel, o.debugLevel); + } +} diff --git a/ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegration.java b/ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegration.java new file mode 100644 index 000000000..9ba03243b --- /dev/null +++ b/ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegration.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.ci; + +import org.openjdk.skara.host.Host; +import org.openjdk.skara.json.JSONObject; +import org.openjdk.skara.json.JSON; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Path; +import java.util.*; + +public interface ContinuousIntegration extends Host { + Job submit(Path source, List jobs, String id) throws IOException; + Job job(String id) throws IOException; + List query(String query) throws IOException; + void cancel(String id) throws IOException; + + static Optional from(URI uri, JSONObject configuration) { + for (var factory : ContinuousIntegrationFactory.factories()) { + var ci = factory.create(uri, configuration); + if (ci.isValid()) { + return Optional.of(ci); + } + } + return Optional.empty(); + } + + static Optional from(URI uri) { + return from(uri, JSON.object()); + } +} diff --git a/ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegrationFactory.java b/ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegrationFactory.java new file mode 100644 index 000000000..1d5f0dcbf --- /dev/null +++ b/ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegrationFactory.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.ci; + +import org.openjdk.skara.json.JSONObject; + +import java.net.URI; +import java.util.*; +import java.util.stream.*; + +public interface ContinuousIntegrationFactory { + ContinuousIntegration create(URI uri, JSONObject configuration); + + static List factories() { + return StreamSupport.stream(ServiceLoader.load(ContinuousIntegrationFactory.class).spliterator(), false) + .collect(Collectors.toList()); + } +} diff --git a/ci/src/main/java/org/openjdk/skara/ci/Job.java b/ci/src/main/java/org/openjdk/skara/ci/Job.java new file mode 100644 index 000000000..ad510a698 --- /dev/null +++ b/ci/src/main/java/org/openjdk/skara/ci/Job.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.ci; + +import java.util.List; + +public interface Job { + static class Status { + private final int numCompleted; + private final int numRunning; + private final int numNotStarted; + + public Status(int numCompleted, int numRunning, int numNotStarted) { + this.numCompleted = numCompleted; + this.numRunning = numRunning; + this.numNotStarted = numNotStarted; + } + + public int numCompleted() { + return numCompleted; + } + + public int numRunning() { + return numRunning; + } + + public int numNotStarted() { + return numNotStarted; + } + + public int numTotal() { + return numCompleted + numRunning + numNotStarted; + } + } + + static class Result { + private final int numPassed; + private final int numFailed; + private final int numSkipped; + + public Result(int numPassed, int numFailed, int numSkipped) { + this.numPassed = numPassed; + this.numFailed = numFailed; + this.numSkipped = numSkipped; + } + + public int numPassed() { + return numPassed; + } + + public int numFailed() { + return numFailed; + } + + public int numSkipped() { + return numSkipped; + } + + public int numTotal() { + return numPassed + numFailed + numSkipped; + } + } + + String id(); + List builds(); + List tests(); + Status status(); + Result result(); + + static enum State { + COMPLETED, + RUNNING, + SCHEDULED + } + State state(); + default boolean isCompleted() { + return state() == State.COMPLETED; + } + default boolean isRunning() { + return state() == State.COMPLETED; + } + default boolean isScheduled() { + return state() == State.SCHEDULED; + } +} diff --git a/ci/src/main/java/org/openjdk/skara/ci/Test.java b/ci/src/main/java/org/openjdk/skara/ci/Test.java new file mode 100644 index 000000000..23ec682e7 --- /dev/null +++ b/ci/src/main/java/org/openjdk/skara/ci/Test.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.ci; + +public class Test { + public static enum Kind { + SINGLE, + GROUP + } + + private final Kind kind; + private final String name; + + public Test(Kind kind, String name) { + this.kind = kind; + this.name = name; + } + + public Kind kind() { + return kind; + } + + public String name() { + return name; + } +} diff --git a/settings.gradle b/settings.gradle index f2b790b16..d2951b31f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -24,6 +24,7 @@ rootProject.name = 'skara' include 'args' include 'bot' +include 'ci' include 'cli' include 'census' include 'email' @@ -54,4 +55,5 @@ include 'bots:mlbridge' include 'bots:notify' include 'bots:pr' include 'bots:submit' +include 'bots:tester' include 'bots:topological' diff --git a/test/build.gradle b/test/build.gradle index 07fc386a8..a9efb9399 100644 --- a/test/build.gradle +++ b/test/build.gradle @@ -26,6 +26,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':json') implementation project(':census') implementation project(':vcs') From 56b9f6517c5a93806ec986674eb730d7ba3ade46 Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Tue, 12 Nov 2019 09:07:06 +0000 Subject: [PATCH 30/54] Update to JDK 13.0.1 Reviewed-by: rwestberg --- bots/cli/build.gradle | 4 ++-- cli/build.gradle | 12 ++++++------ deps.env | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bots/cli/build.gradle b/bots/cli/build.gradle index 9f63dd050..690aeec02 100644 --- a/bots/cli/build.gradle +++ b/bots/cli/build.gradle @@ -83,8 +83,8 @@ images { options = ["--module-path", "plugins"] bundles = ['zip', 'tar.gz'] jdk { - url = 'https://download.java.net/java/GA/jdk12/GPL/openjdk-12_linux-x64_bin.tar.gz' - sha256 = 'b43bc15f4934f6d321170419f2c24451486bc848a2179af5e49d10721438dd56' + url = 'https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_linux-x64_bin.tar.gz' + sha256 = '2e01716546395694d3fad54c9b36d1cd46c5894c06f72d156772efbcf4b41335' } } } diff --git a/cli/build.gradle b/cli/build.gradle index 9da0a98ec..ba885a4c4 100644 --- a/cli/build.gradle +++ b/cli/build.gradle @@ -76,8 +76,8 @@ images { launchers = ext.launchers bundles = ['zip', 'tar.gz'] jdk { - url = 'https://download.java.net/java/GA/jdk12/GPL/openjdk-12_windows-x64_bin.zip' - sha256 = '35a8d018f420fb05fe7c2aa9933122896ca50bd23dbd373e90d8e2f3897c4e92' + url = 'https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_windows-x64_bin.zip' + sha256 = '438a6920f1851b1eeb6f09f05d9f91c4423c6586f7a1a7ccbb19df76ea5901ee' } } @@ -87,8 +87,8 @@ images { man = 'cli/resources/man' bundles = ['zip', 'tar.gz'] jdk { - url = 'https://download.java.net/java/GA/jdk12/GPL/openjdk-12_linux-x64_bin.tar.gz' - sha256 = 'b43bc15f4934f6d321170419f2c24451486bc848a2179af5e49d10721438dd56' + url = 'https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_linux-x64_bin.tar.gz' + sha256 = '2e01716546395694d3fad54c9b36d1cd46c5894c06f72d156772efbcf4b41335' } } @@ -98,8 +98,8 @@ images { man = 'cli/resources/man' bundles = ['zip', 'tar.gz'] jdk { - url = 'https://download.java.net/java/GA/jdk12/GPL/openjdk-12_osx-x64_bin.tar.gz' - sha256 = '52164a04db4d3fdfe128cfc7b868bc4dae52d969f03d53ae9d4239fe783e1a3a' + url = 'https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_osx-x64_bin.tar.gz' + sha256 = '593c5c9dc0978db21b06d6219dc8584b76a59c79d57e6ec1b28ad0d848a7713f' } } diff --git a/deps.env b/deps.env index fc6008567..b7d158e60 100644 --- a/deps.env +++ b/deps.env @@ -1,11 +1,11 @@ -JDK_LINUX_X64_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_linux-x64_bin.tar.gz" -JDK_LINUX_X64_SHA256="b43bc15f4934f6d321170419f2c24451486bc848a2179af5e49d10721438dd56" +JDK_LINUX_X64_URL="https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_linux-x64_bin.tar.gz" +JDK_LINUX_X64_SHA256="2e01716546395694d3fad54c9b36d1cd46c5894c06f72d156772efbcf4b41335" -JDK_MACOS_X64_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_osx-x64_bin.tar.gz" -JDK_MACOS_X64_SHA256="52164a04db4d3fdfe128cfc7b868bc4dae52d969f03d53ae9d4239fe783e1a3a" +JDK_MACOS_X64_URL="https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_osx-x64_bin.tar.gz" +JDK_MACOS_X64_SHA256="593c5c9dc0978db21b06d6219dc8584b76a59c79d57e6ec1b28ad0d848a7713f" -JDK_WINDOWS_X64_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_windows-x64_bin.zip" -JDK_WINDOWS_X64_SHA256="35a8d018f420fb05fe7c2aa9933122896ca50bd23dbd373e90d8e2f3897c4e92" +JDK_WINDOWS_X64_URL="https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_windows-x64_bin.zip" +JDK_WINDOWS_X64_SHA256="438a6920f1851b1eeb6f09f05d9f91c4423c6586f7a1a7ccbb19df76ea5901ee" GRADLE_URL="https://services.gradle.org/distributions/gradle-6.0-bin.zip" GRADLE_SHA256="5a3578b9f0bb162f5e08cf119f447dfb8fa950cedebb4d2a977e912a11a74b91" From e75c0b062d974330cf1273bc502f4a4b95ecfce9 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Tue, 12 Nov 2019 13:18:36 +0000 Subject: [PATCH 31/54] Add pull request prune functionality Reviewed-by: ehelin, kcr --- .../bridgekeeper/BridgekeeperBotFactory.java | 11 +- ...eperBot.java => PullRequestCloserBot.java} | 16 +-- .../bridgekeeper/PullRequestPrunerBot.java | 126 ++++++++++++++++++ ...ts.java => PullRequestCloserBotTests.java} | 6 +- .../PullRequestPrunerBotTests.java | 109 +++++++++++++++ 5 files changed, 254 insertions(+), 14 deletions(-) rename bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/{BridgekeeperBot.java => PullRequestCloserBot.java} (85%) create mode 100644 bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBot.java rename bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/{BridgekeeperBotTests.java => PullRequestCloserBotTests.java} (96%) create mode 100644 bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBotTests.java diff --git a/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotFactory.java b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotFactory.java index 50d8c2b10..9be60c5e6 100644 --- a/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotFactory.java +++ b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotFactory.java @@ -24,6 +24,7 @@ import org.openjdk.skara.bot.*; +import java.time.Duration; import java.util.*; public class BridgekeeperBotFactory implements BotFactory { @@ -37,11 +38,15 @@ public List create(BotConfiguration configuration) { var ret = new ArrayList(); var specific = configuration.specific(); - for (var repo : specific.get("repositories").asArray()) { - var bot = new BridgekeeperBot(configuration.repository(repo.asString())); + for (var repo : specific.get("mirrors").asArray()) { + var bot = new PullRequestCloserBot(configuration.repository(repo.asString())); + ret.add(bot); + } + for (var repo : specific.get("pruned").fields()) { + var maxAge = Duration.parse(repo.value().get("maxage").asString()); + var bot = new PullRequestPrunerBot(configuration.repository(repo.name()), maxAge); ret.add(bot); } - return ret; } } diff --git a/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBot.java b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestCloserBot.java similarity index 85% rename from bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBot.java rename to bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestCloserBot.java index d6210a702..67f0fa759 100644 --- a/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBot.java +++ b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestCloserBot.java @@ -30,19 +30,19 @@ import java.util.function.Consumer; import java.util.logging.Logger; -class BridgekeeperWorkItem implements WorkItem { +class PullRequestCloserBotWorkItem implements WorkItem { private final Logger log = Logger.getLogger("org.openjdk.skara.bots");; private final HostedRepository repository; private final PullRequest pr; private final Consumer errorHandler; - BridgekeeperWorkItem(HostedRepository repository, PullRequest pr, Consumer errorHandler) { + PullRequestCloserBotWorkItem(HostedRepository repository, PullRequest pr, Consumer errorHandler) { this.pr = pr; this.repository = repository; this.errorHandler = errorHandler; } - private final String welcomeMarker = ""; + private final String welcomeMarker = ""; private void checkWelcomeMessage() { log.info("Checking welcome message of " + pr); @@ -69,10 +69,10 @@ private void checkWelcomeMessage() { @Override public boolean concurrentWith(WorkItem other) { - if (!(other instanceof BridgekeeperWorkItem)) { + if (!(other instanceof PullRequestCloserBotWorkItem)) { return true; } - BridgekeeperWorkItem otherItem = (BridgekeeperWorkItem)other; + PullRequestCloserBotWorkItem otherItem = (PullRequestCloserBotWorkItem)other; if (!pr.id().equals(otherItem.pr.id())) { return true; } @@ -93,11 +93,11 @@ public void handleRuntimeException(RuntimeException e) { } } -public class BridgekeeperBot implements Bot { +public class PullRequestCloserBot implements Bot { private final HostedRepository remoteRepo; private final PullRequestUpdateCache updateCache; - BridgekeeperBot(HostedRepository repo) { + PullRequestCloserBot(HostedRepository repo) { this.remoteRepo = repo; this.updateCache = new PullRequestUpdateCache(); } @@ -108,7 +108,7 @@ public List getPeriodicItems() { for (var pr : remoteRepo.pullRequests()) { if (updateCache.needsUpdate(pr)) { - var item = new BridgekeeperWorkItem(remoteRepo, pr, e -> updateCache.invalidate(pr)); + var item = new PullRequestCloserBotWorkItem(remoteRepo, pr, e -> updateCache.invalidate(pr)); ret.add(item); } } diff --git a/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBot.java b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBot.java new file mode 100644 index 000000000..fbe3862f1 --- /dev/null +++ b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBot.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.bridgekeeper; + +import org.openjdk.skara.bot.*; +import org.openjdk.skara.forge.*; + +import java.nio.file.Path; +import java.time.*; +import java.util.*; +import java.util.logging.Logger; + +class PullRequestPrunerBotWorkItem implements WorkItem { + private final Logger log = Logger.getLogger("org.openjdk.skara.bots");; + private final HostedRepository repository; + private final PullRequest pr; + private final Duration maxAge; + + PullRequestPrunerBotWorkItem(HostedRepository repository, PullRequest pr, Duration maxAge) { + this.pr = pr; + this.repository = repository; + this.maxAge = maxAge; + } + + @Override + public boolean concurrentWith(WorkItem other) { + if (!(other instanceof PullRequestPrunerBotWorkItem)) { + return true; + } + PullRequestPrunerBotWorkItem otherItem = (PullRequestPrunerBotWorkItem) other; + if (!pr.id().equals(otherItem.pr.id())) { + return true; + } + if (!repository.name().equals(otherItem.repository.name())) { + return true; + } + return false; + } + + // Prune durations are on the order of days and weeks + private String formatDuration(Duration duration) { + var count = duration.toDays(); + var unit = "day"; + + if (count > 14) { + count /= 7; + unit = "week"; + } + if (count != 1) { + unit += "s"; + } + return count + " " + unit; + } + + private final String noticeMarker = ""; + + @Override + public void run(Path scratchPath) { + var comments = pr.comments(); + if (comments.size() > 0) { + var lastComment = comments.get(comments.size() - 1); + if (lastComment.author().equals(repository.forge().currentUser()) && lastComment.body().contains(noticeMarker)) { + var message = "@" + pr.author().userName() + " This pull request has been inactive for more than " + + formatDuration(maxAge.multipliedBy(2)) + " and will now be automatically closed. If you would " + + "like to continue working on this pull request in the future, feel free to reopen it!"; + log.fine("Posting prune message"); + pr.addComment(message); + pr.setState(PullRequest.State.CLOSED); + return; + } + } + + var message = "@" + pr.author().userName() + " This pull request has been inactive for more than " + + formatDuration(maxAge) + " and will be automatically closed if another " + formatDuration(maxAge) + + " passes without any activity. To avoid this, simply add a new comment to the pull request. Feel free " + + "to ask for assistance if you need help with progressing this pull request towards integration!"; + + log.fine("Posting prune notification message"); + pr.addComment(noticeMarker + "\n\n" + message); + } +} + +public class PullRequestPrunerBot implements Bot { + private final HostedRepository repository; + private final Duration maxAge; + + PullRequestPrunerBot(HostedRepository repository, Duration maxAge) { + this.repository = repository; + this.maxAge = maxAge; + } + + @Override + public List getPeriodicItems() { + List ret = new LinkedList<>(); + var oldestAllowed = ZonedDateTime.now().minus(maxAge); + + for (var pr : repository.pullRequests()) { + if (pr.updatedAt().isBefore(oldestAllowed)) { + var item = new PullRequestPrunerBotWorkItem(repository, pr, maxAge); + ret.add(item); + } + } + + return ret; + } +} diff --git a/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotTests.java b/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestCloserBotTests.java similarity index 96% rename from bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotTests.java rename to bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestCloserBotTests.java index ebc7b8f59..82e5ca8f9 100644 --- a/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotTests.java +++ b/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestCloserBotTests.java @@ -31,13 +31,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -class BridgekeeperBotTests { +class PullRequestCloserBotTests { @Test void simple(TestInfo testInfo) throws IOException { try (var credentials = new HostCredentials(testInfo); var tempFolder = new TemporaryDirectory()) { var author = credentials.getHostedRepository(); - var bot = new BridgekeeperBot(author); + var bot = new PullRequestCloserBot(author); // Populate the projects repository var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); @@ -63,7 +63,7 @@ void keepClosing(TestInfo testInfo) throws IOException { try (var credentials = new HostCredentials(testInfo); var tempFolder = new TemporaryDirectory()) { var author = credentials.getHostedRepository(); - var bot = new BridgekeeperBot(author); + var bot = new PullRequestCloserBot(author); // Populate the projects repository var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); diff --git a/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBotTests.java b/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBotTests.java new file mode 100644 index 000000000..708cdf464 --- /dev/null +++ b/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBotTests.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.bridgekeeper; + +import org.openjdk.skara.test.*; + +import org.junit.jupiter.api.*; + +import java.io.IOException; +import java.time.Duration; + +import static org.junit.jupiter.api.Assertions.*; + +class PullRequestPrunerBotTests { + @Test + void close(TestInfo testInfo) throws IOException, InterruptedException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var bot = new PullRequestPrunerBot(author, Duration.ofMillis(1)); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Make sure the timeout expires + Thread.sleep(100); + + // Let the bot see it - it should give a notice + TestBotRunner.runPeriodicItems(bot); + + assertEquals(1, pr.comments().size()); + assertTrue(pr.comments().get(0).body().contains("will be automatically closed if")); + + pr.addComment("I'm still working on it!"); + + // Make sure the timeout expires again + Thread.sleep(100); + + // Let the bot see it - it should post a second notice + TestBotRunner.runPeriodicItems(bot); + + assertEquals(3, pr.comments().size()); + assertTrue(pr.comments().get(2).body().contains("will be automatically closed if")); + + // Make sure the timeout expires again + Thread.sleep(100); + + // The bot should now close it + TestBotRunner.runPeriodicItems(bot); + + // There should now be no open PRs + var prs = author.pullRequests(); + assertEquals(0, prs.size()); + } + } + + @Test + void dontClose(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var bot = new PullRequestPrunerBot(author, Duration.ofDays(3)); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Let the bot see it + TestBotRunner.runPeriodicItems(bot); + + // There should still be an open PR + var prs = author.pullRequests(); + assertEquals(1, prs.size()); + } + } +} From 592533cd597afcfb214d76ab102980d593febbab Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Tue, 12 Nov 2019 13:55:47 +0000 Subject: [PATCH 32/54] Update messages when a reviewer isn't a project member Reviewed-by: ehelin --- .../java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java index ac1ce5d08..563a14eb6 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java @@ -319,7 +319,7 @@ private String projectRole(Contributor contributor) { } else if (censusInstance.project().isAuthor(contributor.username(), version)) { return "Author"; } - return "none"; + return "no project role"; } void addReview(Review review) { @@ -356,7 +356,7 @@ void addReviewVerdict(Review review) { } var userName = contributor != null ? contributor.username() : review.reviewer().userName() + "@" + censusInstance.namespace().name(); - var userRole = contributor != null ? projectRole(contributor) : "no project role"; + var userRole = contributor != null ? projectRole(contributor) : "no OpenJDK username"; var replyBody = ArchiveMessages.reviewVerdictBody(review.body().orElse(""), review.verdict(), userName, userRole); addReplyCommon(parent, review.reviewer(), subject, replyBody, id); From e60df76c557b27d6d19956f3483ee766306d560a Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Tue, 12 Nov 2019 14:01:48 +0000 Subject: [PATCH 33/54] Add bridgekeeper bot to bot cli image Reviewed-by: ehelin --- bots/cli/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bots/cli/build.gradle b/bots/cli/build.gradle index 690aeec02..6e6c0feb6 100644 --- a/bots/cli/build.gradle +++ b/bots/cli/build.gradle @@ -78,7 +78,8 @@ images { 'org.openjdk.skara.bots.submit', 'org.openjdk.skara.bots.tester', 'org.openjdk.skara.bots.topological', - 'org.openjdk.skara.bots.forward'] + 'org.openjdk.skara.bots.forward', + 'org.openjdk.skara.bots.bridgekeeper'] launchers = ['skara-bots': 'org.openjdk.skara.bots.cli/org.openjdk.skara.bots.cli.BotLauncher'] options = ["--module-path", "plugins"] bundles = ['zip', 'tar.gz'] From f7e71e6235fba2b0f76c7ce28f471c5dd7d918d2 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Tue, 12 Nov 2019 14:36:15 +0000 Subject: [PATCH 34/54] Add missing bridgekeeper dependency Reviewed-by: ehelin --- bots/cli/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/bots/cli/build.gradle b/bots/cli/build.gradle index 6e6c0feb6..3d154e46f 100644 --- a/bots/cli/build.gradle +++ b/bots/cli/build.gradle @@ -47,6 +47,7 @@ dependencies { implementation project(':bots:tester') implementation project(':bots:submit') implementation project(':bots:forward') + implementation project(':bots:bridgekeeper') implementation project(':ci') implementation project(':vcs') implementation project(':jcheck') From 83f70fda63c34cb0f2a3a75c69b75cdc20e12297 Mon Sep 17 00:00:00 2001 From: Christoph Langer Date: Tue, 19 Nov 2019 09:52:34 +0000 Subject: [PATCH 35/54] Rename encoding/src/main/java/org/openjdk/skara/base85/Base85.java Move Base85.java to the right location (encoding/src/main/java/org/openjdk/skara/encoding) Reviewed-by: ehelin --- .../main/java/org/openjdk/skara/{base85 => encoding}/Base85.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename encoding/src/main/java/org/openjdk/skara/{base85 => encoding}/Base85.java (100%) diff --git a/encoding/src/main/java/org/openjdk/skara/base85/Base85.java b/encoding/src/main/java/org/openjdk/skara/encoding/Base85.java similarity index 100% rename from encoding/src/main/java/org/openjdk/skara/base85/Base85.java rename to encoding/src/main/java/org/openjdk/skara/encoding/Base85.java From 47f068b9a6b207fc78ad76dceb70931caa0470c1 Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Tue, 19 Nov 2019 14:23:24 +0000 Subject: [PATCH 36/54] git-sync --fetch is a no-op Reviewed-by: rwestberg --- .../java/org/openjdk/skara/cli/GitSync.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/cli/src/main/java/org/openjdk/skara/cli/GitSync.java b/cli/src/main/java/org/openjdk/skara/cli/GitSync.java index 767b248d5..4b71c8911 100644 --- a/cli/src/main/java/org/openjdk/skara/cli/GitSync.java +++ b/cli/src/main/java/org/openjdk/skara/cli/GitSync.java @@ -38,12 +38,6 @@ private static IOException die(String message) { return new IOException("will never reach here"); } - private static int fetch() throws IOException, InterruptedException { - var pb = new ProcessBuilder("git", "fetch"); - pb.inheritIO(); - return pb.start().waitFor(); - } - private static int pull() throws IOException, InterruptedException { var pb = new ProcessBuilder("git", "pull"); pb.inheritIO(); @@ -71,10 +65,6 @@ public static void main(String[] args) throws IOException, InterruptedException .fullname("pull") .helptext("Pull current branch from origin after successful sync") .optional(), - Switch.shortcut("") - .fullname("fetch") - .helptext("Fetch current branch from origin after successful sync") - .optional(), Switch.shortcut("m") .fullname("mercurial") .helptext("Force use of mercurial") @@ -165,13 +155,6 @@ public static void main(String[] args) throws IOException, InterruptedException System.out.println("done"); } - if (arguments.contains("fetch")) { - int err = fetch(); - if (err != 0) { - System.exit(err); - } - } - if (arguments.contains("pull")) { int err = pull(); if (err != 0) { From 483310ea127795e9ef9a2390f9d90dedf2076d01 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Wed, 20 Nov 2019 12:53:23 +0000 Subject: [PATCH 37/54] Improve MimeText conformance Reviewed-by: ehelin --- .../org/openjdk/skara/email/MimeText.java | 72 +++++++++++++------ .../openjdk/skara/email/MimeTextTests.java | 50 +++++++++++-- 2 files changed, 97 insertions(+), 25 deletions(-) diff --git a/email/src/main/java/org/openjdk/skara/email/MimeText.java b/email/src/main/java/org/openjdk/skara/email/MimeText.java index 34c1fdd0a..323fd69d3 100644 --- a/email/src/main/java/org/openjdk/skara/email/MimeText.java +++ b/email/src/main/java/org/openjdk/skara/email/MimeText.java @@ -24,7 +24,7 @@ import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; -import java.util.Base64; +import java.util.*; import java.util.regex.Pattern; public class MimeText { @@ -33,31 +33,61 @@ public class MimeText { private final static Pattern decodeQuotedPrintablePattern = Pattern.compile("=([0-9A-F]{2})"); public static String encode(String raw) { - var quoteMatcher = encodePattern.matcher(raw); - return quoteMatcher.replaceAll(mo -> "=?UTF-8?B?" + Base64.getEncoder().encodeToString(String.valueOf(mo.group(1)).getBytes(StandardCharsets.UTF_8)) + "?="); + var words = raw.split(" "); + var encodedWords = new ArrayList(); + var lastEncoded = false; + for (var word : words) { + var needsQuotePattern = encodePattern.matcher(word); + if (needsQuotePattern.find()) { + if (lastEncoded) { + // Spaces between encoded words are ignored, so add an explicit one + encodedWords.add("=?UTF-8?B?IA==?="); + } + encodedWords.add("=?UTF-8?B?" + Base64.getEncoder().encodeToString(word.getBytes(StandardCharsets.UTF_8)) + "?="); + lastEncoded = true; + } else { + encodedWords.add(word); + lastEncoded = false; + } + } + return String.join(" ", encodedWords); } public static String decode(String encoded) { + var decoded = new StringBuilder(); var quotedMatcher = decodePattern.matcher(encoded); - return quotedMatcher.replaceAll(mo -> { - try { - if (mo.group(2).toUpperCase().equals("B")) { - return new String(Base64.getDecoder().decode(mo.group(3)), mo.group(1)); - } else { - var quotedPrintableMatcher = decodeQuotedPrintablePattern.matcher(mo.group(3)); - return quotedPrintableMatcher.replaceAll(qmo -> { - var byteValue = new byte[1]; - byteValue[0] = (byte)Integer.parseInt(qmo.group(1), 16); - try { - return new String(byteValue, mo.group(1)); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - }); + var lastMatchEnd = 0; + while (quotedMatcher.find()) { + if (quotedMatcher.start() > lastMatchEnd) { + var separator = encoded.substring(lastMatchEnd, quotedMatcher.start()); + if (!separator.isBlank()) { + decoded.append(separator); + } + } + if (quotedMatcher.group(2).toUpperCase().equals("B")) { + try { + decoded.append(new String(Base64.getDecoder().decode(quotedMatcher.group(3)), quotedMatcher.group(1))); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); } - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); + } else { + var quotedDecodedSpaces = quotedMatcher.group(3).replace("_", " "); + var quotedPrintableMatcher = decodeQuotedPrintablePattern.matcher(quotedDecodedSpaces); + decoded.append(quotedPrintableMatcher.replaceAll(qmo -> { + var byteValue = new byte[1]; + byteValue[0] = (byte)Integer.parseInt(qmo.group(1), 16); + try { + return new String(byteValue, quotedMatcher.group(1)); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + })); } - }); + lastMatchEnd = quotedMatcher.end(); + } + if (lastMatchEnd < encoded.length()) { + decoded.append(encoded, lastMatchEnd, encoded.length()); + } + return decoded.toString(); } } diff --git a/email/src/test/java/org/openjdk/skara/email/MimeTextTests.java b/email/src/test/java/org/openjdk/skara/email/MimeTextTests.java index 2a45506b5..ca6b82635 100644 --- a/email/src/test/java/org/openjdk/skara/email/MimeTextTests.java +++ b/email/src/test/java/org/openjdk/skara/email/MimeTextTests.java @@ -28,17 +28,59 @@ class MimeTextTests { @Test - void encode() { - assertEquals("=?UTF-8?B?w6XDpMO2?=", MimeText.encode("åäö")); + void simple() { + var encoded = "=?UTF-8?B?w6XDpMO2?="; + var decoded = "åäö"; + assertEquals(encoded, MimeText.encode(decoded)); + assertEquals(decoded, MimeText.decode(encoded)); } @Test - void decode() { - assertEquals("åäö", MimeText.decode("=?utf-8?b?w6XDpMO2?=")); + void mixed() { + var encoded = "=?UTF-8?B?VMOpc3Q=?="; + var decoded = "Tést"; + assertEquals(encoded, MimeText.encode(decoded)); + assertEquals(decoded, MimeText.decode(encoded)); + } + + @Test + void multipleWords() { + var encoded = "This is a =?UTF-8?B?dMOpc3Q=?= of =?UTF-8?B?bcO8bHRpcGxl?= words"; + var decoded = "This is a tést of mültiple words"; + assertEquals(encoded, MimeText.encode(decoded)); + assertEquals(decoded, MimeText.decode(encoded)); + } + + @Test + void concatenateTokens() { + var encoded = "=?UTF-8?B?VMOpc3Q=?= =?UTF-8?B?IA==?= =?UTF-8?B?VMOpc3Q=?="; + var decoded = "Tést Tést"; + assertEquals(encoded, MimeText.encode(decoded)); + assertEquals(decoded, MimeText.decode(encoded)); + } + + @Test + void preserveSpaces() { + var encoded = "spac es"; + var decoded = "spac es"; + assertEquals(encoded, MimeText.encode(decoded)); + assertEquals(decoded, MimeText.decode(encoded)); + } + + @Test + void decodeSpaces() { + var encoded = "=?UTF-8?B?VMOpc3Q=?= =?UTF-8?B?VMOpc3Q=?= and "; + var decoded = "TéstTést and "; + assertEquals(decoded, MimeText.decode(encoded)); } @Test void decodeIsoQ() { assertEquals("Bä", MimeText.decode("=?iso-8859-1?Q?B=E4?=")); } + + @Test + void decodeIsoQSpaces() { + assertEquals("Bä Bä Bä", MimeText.decode("=?iso-8859-1?Q?B=E4_B=E4=20B=E4?=")); + } } From 2881d91f048ddc73f06c672babd33a7042ec3783 Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Wed, 20 Nov 2019 14:37:02 +0000 Subject: [PATCH 38/54] Query GitHub for current user when using PAT Reviewed-by: rwestberg --- .../java/org/openjdk/skara/forge/github/GitHubHost.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java index 56c1bc033..b3fafbf79 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java @@ -176,7 +176,10 @@ public HostedRepository repository(String name) { @Override public HostUser user(String username) { var details = request.get("users/" + URLEncoder.encode(username, StandardCharsets.UTF_8)).execute().asObject(); + return asHostUser(details); + } + private static HostUser asHostUser(JSONObject details) { // Always present var login = details.get("login").asString(); var id = details.get("id").asInt(); @@ -196,7 +199,11 @@ public HostUser currentUser() { var appName = appDetails.get("name").asString() + "[bot]"; currentUser = user(appName); } else if (pat != null) { - currentUser = user(pat.username()); + // Cannot always trust username in PAT, e.g. Git Credential Manager + // on Windows always return "PersonalAccessToken" as username. + // Query GitHub for the username instead. + var details = request.get("user").execute().asObject(); + currentUser = asHostUser(details); } else { throw new IllegalStateException("No credentials present"); } From 1a8ea8456976856129b1a31a92debab71985f1e6 Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Thu, 21 Nov 2019 08:45:05 +0000 Subject: [PATCH 39/54] Add config option for git-sync --pull Reviewed-by: rwestberg --- cli/src/main/java/org/openjdk/skara/cli/GitSync.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/src/main/java/org/openjdk/skara/cli/GitSync.java b/cli/src/main/java/org/openjdk/skara/cli/GitSync.java index 4b71c8911..d37242ea0 100644 --- a/cli/src/main/java/org/openjdk/skara/cli/GitSync.java +++ b/cli/src/main/java/org/openjdk/skara/cli/GitSync.java @@ -155,7 +155,12 @@ public static void main(String[] args) throws IOException, InterruptedException System.out.println("done"); } - if (arguments.contains("pull")) { + var shouldPull = arguments.contains("pull"); + if (!shouldPull) { + var lines = repo.config("sync.pull"); + shouldPull = lines.size() == 1 && lines.get(0).toLowerCase().equals("always"); + } + if (shouldPull) { int err = pull(); if (err != 0) { System.exit(err); From 6bea8f3126c8cdefcf92caefc5a484a72bf2c638 Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Thu, 21 Nov 2019 11:00:44 +0000 Subject: [PATCH 40/54] Treat Jira 4xx errors different from 5xx Reviewed-by: ehelin --- .../org/openjdk/skara/issuetracker/jira/JiraProject.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java index 991f868ef..bff9e6457 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java @@ -116,8 +116,11 @@ public Optional issue(String id) { } var issueRequest = request.restrict("issue/" + id); var issue = issueRequest.get("") - .onError(r -> r.statusCode() == 404 ? JSON.object().put("NOT_FOUND", true) : null) + .onError(r -> r.statusCode() < 500 ? JSON.object().put("NOT_FOUND", true) : null) .execute(); + if (issue == null) { + throw new RuntimeException("Server error when trying to fetch issue " + id); + } if (!issue.contains("NOT_FOUND")) { return Optional.of(new JiraIssue(this, issueRequest, issue)); } else { From 2f8cfc903554fc29d762f9c3555073c3b59c3472 Mon Sep 17 00:00:00 2001 From: Erik Helin Date: Fri, 22 Nov 2019 13:13:55 +0000 Subject: [PATCH 41/54] git-sync should use APIs to find upstream Reviewed-by: rwestberg --- .../skara/bot/BotRunnerConfiguration.java | 8 +++++-- .../org/openjdk/skara/bots/pr/CheckRun.java | 4 +++- .../skara/bots/tester/InMemoryHost.java | 4 ++-- .../java/org/openjdk/skara/cli/GitFork.java | 5 +++- .../java/org/openjdk/skara/cli/GitPr.java | 12 +++++++--- .../java/org/openjdk/skara/cli/GitSync.java | 24 +++++++++++++++++-- .../java/org/openjdk/skara/cli/Remote.java | 3 +++ .../java/org/openjdk/skara/forge/Forge.java | 2 +- .../skara/forge/github/GitHubHost.java | 10 +++++--- .../skara/forge/github/GitHubRepository.java | 2 +- .../skara/forge/gitlab/GitLabHost.java | 8 +++++-- .../skara/forge/gitlab/GitLabRepository.java | 2 +- .../org/openjdk/skara/test/CensusBuilder.java | 2 +- .../openjdk/skara/test/HostCredentials.java | 8 +++---- .../java/org/openjdk/skara/test/TestHost.java | 4 ++-- 15 files changed, 72 insertions(+), 26 deletions(-) diff --git a/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java b/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java index 1ea73f919..59f4924c6 100644 --- a/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java +++ b/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java @@ -154,7 +154,9 @@ private Map parseRepositories(JSONObject config) throw throw new ConfigurationError("Repository " + entry.name() + " uses undefined host '" + hostName + "'"); } var host = repositoryHosts.get(hostName); - var repo = host.repository(entry.value().get("repository").asString()); + var repo = host.repository(entry.value().get("repository").asString()).orElseThrow(() -> + new ConfigurationError("Repository " + entry.value().get("repository").asString() + " is not available at " + hostName) + ); ret.put(entry.name(), repo); } @@ -181,7 +183,9 @@ private RepositoryEntry parseRepositoryEntry(String entry) throws ConfigurationE throw new ConfigurationError("Repository entry " + entry + " uses undefined host '" + hostName + "'"); } var repositoryName = entry.substring(hostSeparatorIndex + 1); - ret.repository = host.repository(repositoryName); + ret.repository = host.repository(repositoryName).orElseThrow(() -> + new ConfigurationError("Repository " + repositoryName + " is not available at " + hostName) + ); } else { if (!repositories.containsKey(entry)) { throw new ConfigurationError("Repository " + entry + " is not defined!"); diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java index 7ce7e3e4e..c671112f3 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java @@ -137,7 +137,9 @@ private List botSpecificChecks() throws IOException { var sourceBranch = mergeSourceBranch(); if (sourceBranch.isPresent() && sourceRepo.isPresent()) { try { - var mergeSourceRepo = pr.repository().forge().repository(sourceRepo.get()); + var mergeSourceRepo = pr.repository().forge().repository(sourceRepo.get()).orElseThrow(() -> + new RuntimeException("Could not find repository " + sourceRepo.get()) + ); try { var sourceHash = prInstance.localRepo().fetch(mergeSourceRepo.url(), sourceBranch.get()); if (!prInstance.localRepo().isAncestor(commits.get(1).hash(), sourceHash)) { diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHost.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHost.java index ae7f2008b..acda49f91 100644 --- a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHost.java +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHost.java @@ -37,8 +37,8 @@ public boolean isValid() { } @Override - public HostedRepository repository(String name) { - return null; + public Optional repository(String name) { + return Optional.empty(); } @Override diff --git a/cli/src/main/java/org/openjdk/skara/cli/GitFork.java b/cli/src/main/java/org/openjdk/skara/cli/GitFork.java index 01e140ecf..27f965430 100644 --- a/cli/src/main/java/org/openjdk/skara/cli/GitFork.java +++ b/cli/src/main/java/org/openjdk/skara/cli/GitFork.java @@ -157,7 +157,10 @@ public static void main(String[] args) throws IOException { path = path.substring(1); } - var fork = host.get().repository(path).fork(); + var hostedRepo = host.get().repository(path).orElseThrow(() -> + new IOException("Could not find repository at " + uri.toString()) + ); + var fork = hostedRepo.fork(); if (token == null) { GitCredentials.approve(credentials); diff --git a/cli/src/main/java/org/openjdk/skara/cli/GitPr.java b/cli/src/main/java/org/openjdk/skara/cli/GitPr.java index c1bf9cd8c..dfddea9e0 100644 --- a/cli/src/main/java/org/openjdk/skara/cli/GitPr.java +++ b/cli/src/main/java/org/openjdk/skara/cli/GitPr.java @@ -108,7 +108,9 @@ private static HostedRepository getHostedRepositoryFor(URI uri, GitCredentials c if (host.isEmpty() || !host.get().isValid()) { exit("error: failed to connect to host " + uri); } - var remoteRepo = host.get().repository(projectName(uri)); + var remoteRepo = host.get().repository(projectName(uri)).orElseThrow(() -> + new IOException("Could not find repository at: " + uri.toString()) + ); var parentRepo = remoteRepo.parent(); var targetRepo = parentRepo.isPresent() ? parentRepo.get() : remoteRepo; return targetRepo; @@ -418,7 +420,9 @@ public static void main(String[] args) throws IOException, InterruptedException System.exit(1); } - var remoteRepo = host.get().repository(projectName(uri)); + var remoteRepo = host.get().repository(projectName(uri)).orElseThrow(() -> + new IOException("Could not find repository at " + uri.toString()) + ); if (token == null) { GitCredentials.approve(credentials); } @@ -575,7 +579,9 @@ public static void main(String[] args) throws IOException, InterruptedException System.exit(1); } - var remoteRepo = host.get().repository(projectName(uri)); + var remoteRepo = host.get().repository(projectName(uri)).orElseThrow(() -> + new IOException("Could not find repository at " + uri.toString()) + ); if (token == null) { GitCredentials.approve(credentials); } diff --git a/cli/src/main/java/org/openjdk/skara/cli/GitSync.java b/cli/src/main/java/org/openjdk/skara/cli/GitSync.java index d37242ea0..1f0c732f5 100644 --- a/cli/src/main/java/org/openjdk/skara/cli/GitSync.java +++ b/cli/src/main/java/org/openjdk/skara/cli/GitSync.java @@ -24,9 +24,11 @@ import org.openjdk.skara.args.*; import org.openjdk.skara.vcs.*; +import org.openjdk.skara.forge.*; +import org.openjdk.skara.proxy.HttpProxy; import java.io.*; -import java.net.URI; +import java.net.*; import java.nio.file.*; import java.util.*; import java.util.logging.*; @@ -101,6 +103,8 @@ public static void main(String[] args) throws IOException, InterruptedException die("error: no repository found at " + cwd.toString()) ); + HttpProxy.setup(); + var remotes = repo.remotes(); String upstream = null; @@ -111,9 +115,25 @@ public static void main(String[] args) throws IOException, InterruptedException if (lines.size() == 1 && remotes.contains(lines.get(0))) { upstream = lines.get(0); } else { - die("No remote provided to fetch from, please set the --from flag"); + if (remotes.contains("origin")) { + var originPullPath = repo.pullPath("origin"); + try { + var uri = Remote.toWebURI(originPullPath); + upstream = Forge.from(URI.create(uri.getScheme() + "://" + uri.getHost())) + .flatMap(f -> f.repository(uri.getPath().substring(1))) + .flatMap(r -> r.parent()) + .map(p -> p.webUrl().toString()) + .orElse(null); + } catch (IllegalArgumentException e) { + upstream = null; + } + } } } + + if (upstream == null) { + die("Could not find upstream repository, please specify one with --from"); + } var upstreamPullPath = remotes.contains(upstream) ? Remote.toURI(repo.pullPath(upstream)) : URI.create(upstream); diff --git a/cli/src/main/java/org/openjdk/skara/cli/Remote.java b/cli/src/main/java/org/openjdk/skara/cli/Remote.java index 334f597cd..c71f3ca6e 100644 --- a/cli/src/main/java/org/openjdk/skara/cli/Remote.java +++ b/cli/src/main/java/org/openjdk/skara/cli/Remote.java @@ -34,6 +34,9 @@ public static URI toWebURI(String remotePath) throws IOException { if (remotePath.startsWith("git+")) { remotePath = remotePath.substring("git+".length()); } + if (remotePath.endsWith(".git")) { + remotePath = remotePath.substring(0, remotePath.length() - ".git".length()); + } if (remotePath.startsWith("http")) { return URI.create(remotePath); } else { diff --git a/forge/src/main/java/org/openjdk/skara/forge/Forge.java b/forge/src/main/java/org/openjdk/skara/forge/Forge.java index 90dbc58bc..6f7b9a514 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/Forge.java +++ b/forge/src/main/java/org/openjdk/skara/forge/Forge.java @@ -30,7 +30,7 @@ import java.util.stream.Collectors; public interface Forge extends Host { - HostedRepository repository(String name); + Optional repository(String name); boolean supportsReviewBody(); static Forge from(String name, URI uri, Credential credential, JSONObject configuration) { diff --git a/forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java index b3fafbf79..adcf15083 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java @@ -30,7 +30,7 @@ import java.io.IOException; import java.net.*; import java.nio.charset.StandardCharsets; -import java.util.Arrays; +import java.util.*; import java.util.logging.Logger; import java.util.regex.Pattern; @@ -169,8 +169,12 @@ JSONObject runSearch(String query) { } @Override - public HostedRepository repository(String name) { - return new GitHubRepository(this, name); + public Optional repository(String name) { + try { + return Optional.of(new GitHubRepository(this, name)); + } catch (Throwable t) { + return Optional.empty(); + } } @Override diff --git a/forge/src/main/java/org/openjdk/skara/forge/github/GitHubRepository.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubRepository.java index ff1980fe5..c89301d79 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/github/GitHubRepository.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubRepository.java @@ -190,7 +190,7 @@ public Optional parseWebHook(JSONValue body) { @Override public HostedRepository fork() { var response = request.post("forks").execute(); - return gitHubHost.repository(response.get("full_name").asString()); + return gitHubHost.repository(response.get("full_name").asString()).orElseThrow(RuntimeException::new); } @Override diff --git a/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabHost.java b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabHost.java index 71f450926..6cb5029a3 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabHost.java +++ b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabHost.java @@ -102,8 +102,12 @@ JSONObject getProjectInfo(String name) { } @Override - public HostedRepository repository(String name) { - return new GitLabRepository(this, name); + public Optional repository(String name) { + try { + return Optional.of(new GitLabRepository(this, name)); + } catch (Throwable t) { + return Optional.empty(); + } } private HostUser parseUserDetails(JSONObject details) { diff --git a/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabRepository.java b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabRepository.java index 553d4e3e1..faa52ec5a 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabRepository.java +++ b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabRepository.java @@ -235,7 +235,7 @@ public HostedRepository fork() { e.printStackTrace(); } } - return gitLabHost.repository(forkedRepoName); + return gitLabHost.repository(forkedRepoName).orElseThrow(RuntimeException::new); } @Override diff --git a/test/src/main/java/org/openjdk/skara/test/CensusBuilder.java b/test/src/main/java/org/openjdk/skara/test/CensusBuilder.java index 6d4196962..7780d2ae7 100644 --- a/test/src/main/java/org/openjdk/skara/test/CensusBuilder.java +++ b/test/src/main/java/org/openjdk/skara/test/CensusBuilder.java @@ -208,7 +208,7 @@ private void generateVersion(Path folder) throws IOException { public HostedRepository build() { try { var host = TestHost.createNew(List.of(new HostUser(1, "cu", "Census User"))); - var repository = host.repository("census"); + var repository = host.repository("census").orElseThrow(); var folder = Files.createTempDirectory("censusbuilder"); var localRepository = Repository.init(folder, VCS.GIT); diff --git a/test/src/main/java/org/openjdk/skara/test/HostCredentials.java b/test/src/main/java/org/openjdk/skara/test/HostCredentials.java index 9ab0d76d7..ee7a38a72 100644 --- a/test/src/main/java/org/openjdk/skara/test/HostCredentials.java +++ b/test/src/main/java/org/openjdk/skara/test/HostCredentials.java @@ -91,7 +91,7 @@ public IssueTracker createIssueHost(int userIndex) { @Override public HostedRepository getHostedRepository(Forge host) { - return host.repository(config.get("project").asString()); + return host.repository(config.get("project").asString()).orElseThrow(); } @Override @@ -128,7 +128,7 @@ public IssueTracker createIssueHost(int userIndex) { @Override public HostedRepository getHostedRepository(Forge host) { - return host.repository(config.get("project").asString()); + return host.repository(config.get("project").asString()).orElseThrow(); } @Override @@ -165,7 +165,7 @@ public IssueTracker createIssueHost(int userIndex) { @Override public HostedRepository getHostedRepository(Forge host) { - return host.repository(config.get("project").asString()); + return host.repository(config.get("project").asString()).orElseThrow(); } @Override @@ -209,7 +209,7 @@ public IssueTracker createIssueHost(int userIndex) { @Override public HostedRepository getHostedRepository(Forge host) { - return host.repository("test"); + return host.repository("test").orElseThrow(); } @Override diff --git a/test/src/main/java/org/openjdk/skara/test/TestHost.java b/test/src/main/java/org/openjdk/skara/test/TestHost.java index caedb9cea..6bdc3b0fc 100644 --- a/test/src/main/java/org/openjdk/skara/test/TestHost.java +++ b/test/src/main/java/org/openjdk/skara/test/TestHost.java @@ -85,7 +85,7 @@ public boolean isValid() { } @Override - public HostedRepository repository(String name) { + public Optional repository(String name) { Repository localRepository; if (data.repositories.containsKey(name)) { localRepository = data.repositories.get(name); @@ -96,7 +96,7 @@ public HostedRepository repository(String name) { localRepository = createLocalRepository(); data.repositories.put(name, localRepository); } - return new TestHostedRepository(this, name, localRepository); + return Optional.of(new TestHostedRepository(this, name, localRepository)); } @Override From a77b854b234fba3b1f1e8dd824b0e9e1dee393db Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Fri, 22 Nov 2019 13:52:53 +0000 Subject: [PATCH 42/54] 166: Add `/solves` command Reviewed-by: ehelin --- .../org/openjdk/skara/bots/pr/CheckRun.java | 2 +- .../skara/bots/pr/CommandWorkItem.java | 3 +- .../skara/bots/pr/PullRequestInstance.java | 6 +- .../openjdk/skara/bots/pr/SolvesCommand.java | 99 ++++++++ .../openjdk/skara/bots/pr/SolvesTracker.java | 67 +++++ .../openjdk/skara/bots/pr/SolvesTests.java | 232 ++++++++++++++++++ .../openjdk/skara/bots/pr/SummaryTests.java | 1 - 7 files changed, 406 insertions(+), 4 deletions(-) create mode 100644 bots/pr/src/main/java/org/openjdk/skara/bots/pr/SolvesCommand.java create mode 100644 bots/pr/src/main/java/org/openjdk/skara/bots/pr/SolvesTracker.java create mode 100644 bots/pr/src/test/java/org/openjdk/skara/bots/pr/SolvesTests.java diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java index c671112f3..d8842fe9a 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java @@ -282,7 +282,7 @@ private String getStatusMessage(List reviews, PullRequestCheckIssueVisit progressBody.append("\n"); } else { progressBody.append("⚠️ Failed to retrieve information on issue `"); - progressBody.append(issue.get().toString()); + progressBody.append(issue.get().id()); progressBody.append("`.\n"); } } diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CommandWorkItem.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CommandWorkItem.java index 526216d49..3fb2e9f70 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CommandWorkItem.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CommandWorkItem.java @@ -48,7 +48,8 @@ public class CommandWorkItem extends PullRequestWorkItem { "integrate", new IntegrateCommand(), "sponsor", new SponsorCommand(), "contributor", new ContributorCommand(), - "summary", new SummaryCommand() + "summary", new SummaryCommand(), + "solves", new SolvesCommand() ); static class HelpCommand implements CommandHandler { diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestInstance.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestInstance.java index a84fcef9a..86561bfd2 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestInstance.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/PullRequestInstance.java @@ -82,11 +82,15 @@ private String commitMessage(List activeReviews, Namespace namespace, bo .map(email -> Author.fromString(email.toString())) .collect(Collectors.toList()); + var additionalIssues = SolvesTracker.currentSolved(pr.repository().forge().currentUser(), comments); var summary = Summary.summary(pr.repository().forge().currentUser(), comments); var issue = Issue.fromString(pr.title()); var commitMessageBuilder = issue.map(CommitMessage::title).orElseGet(() -> CommitMessage.title(isMerge ? "Merge" : pr.title())); + if (issue.isPresent()) { + commitMessageBuilder.issues(additionalIssues); + } commitMessageBuilder.contributors(additionalContributors) - .reviewers(reviewers); + .reviewers(reviewers); summary.ifPresent(commitMessageBuilder::summary); return String.join("\n", commitMessageBuilder.format(CommitMessageFormatters.v1)); diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SolvesCommand.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SolvesCommand.java new file mode 100644 index 000000000..0c12b82aa --- /dev/null +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SolvesCommand.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.pr; + +import org.openjdk.skara.forge.PullRequest; +import org.openjdk.skara.issuetracker.Comment; +import org.openjdk.skara.vcs.openjdk.Issue; + +import java.io.PrintWriter; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; + +public class SolvesCommand implements CommandHandler { + private void showHelp(PrintWriter reply) { + reply.println("To add an additional issue to the list of issues that this PR solves: `/solves : `." + + "To remove a previously added additional issue: `/solves `."); + } + + @Override + public void handle(PullRequest pr, CensusInstance censusInstance, Path scratchPath, String args, Comment comment, List allComments, PrintWriter reply) { + if (!comment.author().equals(pr.author())) { + reply.println("Only the author (@" + pr.author().userName() + ") is allowed to issue the `solves` command."); + return; + } + + if (args.isBlank()) { + showHelp(reply); + return; + } + + var currentSolved = SolvesTracker.currentSolved(pr.repository().forge().currentUser(), allComments) + .stream() + .map(Issue::id) + .collect(Collectors.toSet()); + + var issue = Issue.fromString(args); + if (issue.isEmpty()) { + issue = Issue.fromString(args + ": deleteme"); + if (issue.isEmpty()) { + reply.println("Invalid command syntax."); + showHelp(reply); + return; + } + + if (currentSolved.contains(issue.get().id())) { + reply.println(SolvesTracker.removeSolvesMarker(issue.get()));; + reply.println("Removing additional issue from solves list: `" + issue.get().id() + "`."); + } else { + reply.println("Could not find issue `" + issue.get().id() + "` in the list of additional solved issues."); + } + return; + } + + var titleIssue = Issue.fromString(pr.title()); + if (titleIssue.isEmpty()) { + reply.print("The primary solved issue for a PR is set through the PR title. Since the current title does "); + reply.println("not contain an issue reference, it will now be updated."); + pr.setTitle(issue.get().toString()); + return; + } + if (titleIssue.get().id().equals(issue.get().id())) { + reply.println("This issue is referenced in the PR title - it will now be updated."); + pr.setTitle(issue.get().toString()); + return; + } + reply.println(SolvesTracker.setSolvesMarker(issue.get())); + if (currentSolved.contains(issue.get().id())) { + reply.println("Updating description of additional solved issue: `" + issue.get().toString() + "`."); + } else { + reply.println("Adding additional issue to solves list: `" + issue.get().toString() + "`."); + } + } + + @Override + public String description() { + return "add an additional issue that this PR solves"; + } +} diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SolvesTracker.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SolvesTracker.java new file mode 100644 index 000000000..98ee9a9e7 --- /dev/null +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SolvesTracker.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.pr; + +import org.openjdk.skara.host.HostUser; +import org.openjdk.skara.issuetracker.Comment; +import org.openjdk.skara.vcs.openjdk.Issue; + +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.regex.*; +import java.util.stream.Collectors; + +public class SolvesTracker { + private final static String solvesMarker = ""; + private final static Pattern markerPattern = Pattern.compile(""); + + static String setSolvesMarker(Issue issue) { + var encodedDescription = Base64.getEncoder().encodeToString(issue.description().getBytes(StandardCharsets.UTF_8)); + return String.format(solvesMarker, issue.id(), encodedDescription); + } + + static String removeSolvesMarker(Issue issue) { + return String.format(solvesMarker, issue.id(), ""); + } + + static List currentSolved(HostUser botUser, List comments) { + var solvesActions = comments.stream() + .filter(comment -> comment.author().equals(botUser)) + .map(comment -> markerPattern.matcher(comment.body())) + .filter(Matcher::find) + .collect(Collectors.toList()); + var current = new LinkedHashMap(); + for (var action : solvesActions) { + var key = action.group(1); + if (action.group(2).equals("")) { + current.remove(key); + } else { + var decodedDescription = new String(Base64.getDecoder().decode(action.group(2)), StandardCharsets.UTF_8); + var issue = new Issue(key, decodedDescription); + current.put(key, issue); + } + } + + return new ArrayList<>(current.values()); + } +} diff --git a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SolvesTests.java b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SolvesTests.java new file mode 100644 index 000000000..313710d6d --- /dev/null +++ b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SolvesTests.java @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.pr; + +import org.junit.jupiter.api.*; +import org.openjdk.skara.forge.Review; +import org.openjdk.skara.issuetracker.Comment; +import org.openjdk.skara.test.*; +import org.openjdk.skara.vcs.Repository; + +import java.io.IOException; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.openjdk.skara.bots.pr.PullRequestAsserts.assertLastCommentContains; + +class SolvesTests { + @Test + void simple(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var integrator = credentials.getHostedRepository(); + + var censusBuilder = credentials.getCensusBuilder() + .addReviewer(integrator.forge().currentUser().id()) + .addCommitter(author.forge().currentUser().id()); + var prBot = new PullRequestBot(integrator, censusBuilder.build(), "master"); + + // Populate the projects repository + var localRepoFolder = tempFolder.path().resolve("localrepo"); + var localRepo = CheckableRepository.init(localRepoFolder, author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + assertFalse(CheckableRepository.hasBeenEdited(localRepo)); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "123: This is a pull request"); + + // No arguments + pr.addComment("/solves"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a help message + assertLastCommentContains(pr,"To add an additional"); + + // Invalid syntax + pr.addComment("/solves something I guess"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a failure message + assertLastCommentContains(pr,"Invalid"); + + // Add an issue + pr.addComment("/solves 1234: An issue"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a success message + assertLastCommentContains(pr,"Adding additional"); + + // Try to remove a not-previously-added issue + pr.addComment("/solves 1235"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a failure message + assertLastCommentContains(pr,"Could not find"); + + // Now remove the added one + pr.addComment("/solves 1234"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a success message + assertLastCommentContains(pr,"Removing additional"); + + // Add two more issues + pr.addComment("/solves 12345: Another issue"); + pr.addComment("/solves 123456: Yet another issue"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a success message + assertLastCommentContains(pr,"Adding additional"); + + // Update the description of the first one + pr.addComment("/solves 12345: This is indeed another issue"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a success message + assertLastCommentContains(pr,"Updating description"); + + // Approve it as another user + var approvalPr = integrator.pullRequest(pr.id()); + approvalPr.addReview(Review.Verdict.APPROVED, "Approved"); + TestBotRunner.runPeriodicItems(prBot); + TestBotRunner.runPeriodicItems(prBot); + + // The commit message preview should contain the additional issues + var preview = pr.comments().stream() + .filter(comment -> comment.body().contains("The commit message will be")) + .map(Comment::body) + .findFirst() + .orElseThrow(); + assertTrue(preview.contains("123: This is a pull request")); + assertTrue(preview.contains("12345: This is indeed another issue")); + assertTrue(preview.contains("123456: Yet another issue")); + + // Integrate + pr.addComment("/integrate"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with an ok message + assertLastCommentContains(pr,"Pushed as commit"); + + // The change should now be present on the master branch + var pushedFolder = tempFolder.path().resolve("pushed"); + var pushedRepo = Repository.materialize(pushedFolder, author.url(), "master"); + assertTrue(CheckableRepository.hasBeenEdited(pushedRepo)); + + var headHash = pushedRepo.resolve("HEAD").orElseThrow(); + var headCommit = pushedRepo.commits(headHash.hex() + "^.." + headHash.hex()).asList().get(0); + + // The additional issues should be present in the commit message + assertEquals(List.of("123: This is a pull request", + "12345: This is indeed another issue", + "123456: Yet another issue", + "", + "Reviewed-by: integrationreviewer1"), headCommit.message()); + } + } + + @Test + void invalidCommandAuthor(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var integrator = credentials.getHostedRepository(); + var external = credentials.getHostedRepository(); + + var censusBuilder = credentials.getCensusBuilder() + .addAuthor(author.forge().currentUser().id()); + var mergeBot = new PullRequestBot(integrator, censusBuilder.build(), "master"); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + assertFalse(CheckableRepository.hasBeenEdited(localRepo)); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Issue a solves command not as the PR author + var externalPr = external.pullRequest(pr.id()); + externalPr.addComment("/solves 1234: an issue"); + TestBotRunner.runPeriodicItems(mergeBot); + + // The bot should reply with an error message + var error = pr.comments().stream() + .filter(comment -> comment.body().contains("Only the author")) + .count(); + assertEquals(1, error); + } + } + + @Test + void issueInTitle(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var integrator = credentials.getHostedRepository(); + var external = credentials.getHostedRepository(); + + var censusBuilder = credentials.getCensusBuilder() + .addAuthor(author.forge().currentUser().id()); + var prBot = new PullRequestBot(integrator, censusBuilder.build(), "master"); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + assertFalse(CheckableRepository.hasBeenEdited(localRepo)); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Add an issue + pr.addComment("/solves 1234: An issue"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a success message + assertLastCommentContains(pr,"current title"); + + var updatedPr = author.pullRequest(pr.id()); + assertEquals("1234: An issue", updatedPr.title()); + + // Update the issue description + pr.addComment("/solves 1234: Yes this is an issue"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a success message + assertLastCommentContains(pr,"will now be updated"); + + updatedPr = author.pullRequest(pr.id()); + assertEquals("1234: Yes this is an issue", updatedPr.title()); + } + } +} diff --git a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SummaryTests.java b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SummaryTests.java index 8b16e103c..467a698a2 100644 --- a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SummaryTests.java +++ b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SummaryTests.java @@ -168,5 +168,4 @@ void invalidCommandAuthor(TestInfo testInfo) throws IOException { assertEquals(1, error); } } - } From 943987fb756720c098b809cd09d01a3a6883532c Mon Sep 17 00:00:00 2001 From: Robin Westberg Date: Mon, 25 Nov 2019 07:00:52 +0000 Subject: [PATCH 43/54] 166: Update PR status in body with multiple issues if needed Reviewed-by: ehelin --- .../org/openjdk/skara/bots/pr/CheckRun.java | 41 +++++++++------ .../openjdk/skara/bots/pr/CheckWorkItem.java | 2 +- .../openjdk/skara/bots/pr/SolvesTests.java | 51 ++++++++++++++++++- 3 files changed, 75 insertions(+), 19 deletions(-) diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java index d8842fe9a..54945fb2b 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java @@ -263,27 +263,36 @@ private Optional getReviewersList(List reviews) { } } - private String getStatusMessage(List reviews, PullRequestCheckIssueVisitor visitor) { + private String getStatusMessage(List comments, List reviews, PullRequestCheckIssueVisitor visitor) { var progressBody = new StringBuilder(); progressBody.append("## Progress\n"); progressBody.append(getChecksList(visitor)); var issue = Issue.fromString(pr.title()); if (issueProject != null && issue.isPresent()) { - progressBody.append("\n\n## Issue\n"); - var iss = issueProject.issue(issue.get().id()); - if (iss.isPresent()) { - progressBody.append("["); - progressBody.append(iss.get().id()); - progressBody.append("]("); - progressBody.append(iss.get().webUrl()); - progressBody.append("): "); - progressBody.append(iss.get().title()); - progressBody.append("\n"); - } else { - progressBody.append("⚠️ Failed to retrieve information on issue `"); - progressBody.append(issue.get().id()); - progressBody.append("`.\n"); + var allIssues = new ArrayList(); + allIssues.add(issue.get()); + allIssues.addAll(SolvesTracker.currentSolved(pr.repository().forge().currentUser(), comments)); + progressBody.append("\n\n## Issue"); + if (allIssues.size() > 1) { + progressBody.append("s"); + } + progressBody.append("\n"); + for (var currentIssue : allIssues) { + var iss = issueProject.issue(currentIssue.id()); + if (iss.isPresent()) { + progressBody.append("["); + progressBody.append(iss.get().id()); + progressBody.append("]("); + progressBody.append(iss.get().webUrl()); + progressBody.append("): "); + progressBody.append(iss.get().title()); + progressBody.append("\n"); + } else { + progressBody.append("⚠️ Failed to retrieve information on issue `"); + progressBody.append(currentIssue.id()); + progressBody.append("`.\n"); + } } } @@ -477,7 +486,7 @@ private void checkStatus() { updateReadyForReview(visitor, additionalErrors); // Calculate and update the status message if needed - var statusMessage = getStatusMessage(activeReviews, visitor); + var statusMessage = getStatusMessage(comments, activeReviews, visitor); var updatedBody = updateStatusMessage(statusMessage); // Post / update approval messages (only needed if the review itself can't contain a body) diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckWorkItem.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckWorkItem.java index 980965326..ed6ae8e4d 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckWorkItem.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckWorkItem.java @@ -44,7 +44,7 @@ class CheckWorkItem extends PullRequestWorkItem { private final Map blockingLabels; private final IssueProject issueProject; - private final Pattern metadataComments = Pattern.compile("