diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9ea9a0c6..abb9c202b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,3 +25,4 @@ jobs: - uses: actions/checkout@v1 - name: Build and test run: gradlew.bat test --info --stacktrace + shell: cmd diff --git a/README.md b/README.md index 69cd87c17..f934e335a 100644 --- a/README.md +++ b/README.md @@ -39,14 +39,14 @@ external Git source code hosting providers are available: ## Building [JDK 12](http://jdk.java.net/12/) or later and [Gradle](https://gradle.org/) -5.2.1 or later is required for building. To build the project on macOS or -GNU/Linux, just run the following command from the source tree root: +5.6.2 or later is required for building. To build the project on macOS or +GNU/Linux x64, just run the following command from the source tree root: ```bash $ sh gradlew ``` -To build the project on Windows, run the following command from the source tree root: +To build the project on Windows x64, run the following command from the source tree root: ```bat > gradlew @@ -55,6 +55,42 @@ To build the project on Windows, run the following command from the source tree The extracted jlinked image will end up in the `build` directory in the source tree root. +### Other operating systems and CPU architectures + +If you want to build on an operating system other than GNU/Linux, macOS or +Windows _or_ if you want to build on a CPU architecture other than x64, then +ensure that you have JDK 12 or later installed locally. You can then run the +following command from the source tree root: + +```bash +$ sh gradlew +``` + +The extracted jlinked image will end up in the `build` directory in the source +tree root. + +### Offline builds + +If you don't want the build to automatically download any dependencies, then +you must ensure that you have installed the following software locally: + +- JDK 12 or later +- Gradle 5.6.2 or later + +To create a build then run the command: + +```bash +$ gradle offline +``` + +_Please note_ that the above command does _not_ make use of `gradlew` to avoid +downloading Gradle. + +The extracted jlinked image will end up in the `build` directory in the source +tree root. + +### Cross-linking + It is also supported to cross-jlink jimages to GNU/Linux, macOS and/or Windows from any of the aforementioned operating systems. To build all applicable jimages (including the server-side tooling), run the following command from the diff --git a/bot/build.gradle b/bot/build.gradle index 46bbe201d..5e5c54b8d 100644 --- a/bot/build.gradle +++ b/bot/build.gradle @@ -30,6 +30,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':host') implementation project(':network') implementation project(':issuetracker') diff --git a/bot/src/main/java/module-info.java b/bot/src/main/java/module-info.java index 373540905..92b775627 100644 --- a/bot/src/main/java/module-info.java +++ b/bot/src/main/java/module-info.java @@ -21,6 +21,7 @@ * questions. */ module org.openjdk.skara.bot { + requires transitive org.openjdk.skara.ci; requires transitive org.openjdk.skara.host; requires transitive org.openjdk.skara.issuetracker; requires transitive org.openjdk.skara.forge; diff --git a/bot/src/main/java/org/openjdk/skara/bot/BotConfiguration.java b/bot/src/main/java/org/openjdk/skara/bot/BotConfiguration.java index 448c1fc7d..25a2aaaa0 100644 --- a/bot/src/main/java/org/openjdk/skara/bot/BotConfiguration.java +++ b/bot/src/main/java/org/openjdk/skara/bot/BotConfiguration.java @@ -22,6 +22,7 @@ */ package org.openjdk.skara.bot; +import org.openjdk.skara.ci.ContinuousIntegration; import org.openjdk.skara.forge.HostedRepository; import org.openjdk.skara.issuetracker.IssueProject; import org.openjdk.skara.json.JSONObject; @@ -49,6 +50,13 @@ public interface BotConfiguration { */ IssueProject issueProject(String name); + /** + * Configuration-specific name mapped to a ContinuousIntegration. + * @param name + * @return + */ + ContinuousIntegration continuousIntegration(String name); + /** * Retrieves the ref name that optionally follows the configuration-specific repository name. * If not configured, returns the name of the VCS default branch. diff --git a/bot/src/main/java/org/openjdk/skara/bot/BotRunner.java b/bot/src/main/java/org/openjdk/skara/bot/BotRunner.java index 1cb6d1592..81c2da47f 100644 --- a/bot/src/main/java/org/openjdk/skara/bot/BotRunner.java +++ b/bot/src/main/java/org/openjdk/skara/bot/BotRunner.java @@ -177,7 +177,6 @@ private void drain(Duration timeout) throws TimeoutException { } try { Thread.sleep(1); - watchdog(); } catch (InterruptedException e) { log.warning("Exception during queue drain"); log.throwing("BotRunner", "drain", e); @@ -259,7 +258,10 @@ private void processRestRequest(JSONValue request) { } public void run() { - log.info("Starting BotRunner execution, will run forever."); + run(Duration.ofDays(10 * 365)); + } + + public void run(Duration timeout) { log.info("Periodic task interval: " + config.scheduledExecutionPeriod()); log.info("Concurrency: " + config.concurrency()); @@ -280,7 +282,7 @@ public void run() { config.scheduledExecutionPeriod().toMillis(), TimeUnit.MILLISECONDS); try { - executor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + executor.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS); } catch (InterruptedException e) { e.printStackTrace(); } diff --git a/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java b/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java index 410dfdc00..7668fb376 100644 --- a/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java +++ b/bot/src/main/java/org/openjdk/skara/bot/BotRunnerConfiguration.java @@ -22,26 +22,28 @@ */ package org.openjdk.skara.bot; +import org.openjdk.skara.ci.ContinuousIntegration; import org.openjdk.skara.forge.*; -import org.openjdk.skara.host.*; +import org.openjdk.skara.host.Credential; import org.openjdk.skara.issuetracker.*; -import org.openjdk.skara.network.URIBuilder; import org.openjdk.skara.json.JSONObject; +import org.openjdk.skara.network.URIBuilder; import org.openjdk.skara.vcs.VCS; import java.io.*; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.time.Duration; import java.util.*; import java.util.logging.Logger; -import java.util.regex.Pattern; public class BotRunnerConfiguration { private final Logger log; private final JSONObject config; private final Map repositoryHosts; private final Map issueHosts; + private final Map continuousIntegrations; private final Map repositories; private BotRunnerConfiguration(JSONObject config, Path cwd) throws ConfigurationError { @@ -50,6 +52,7 @@ private BotRunnerConfiguration(JSONObject config, Path cwd) throws Configuration repositoryHosts = parseRepositoryHosts(config, cwd); issueHosts = parseIssueHosts(config, cwd); + continuousIntegrations = parseContinuousIntegrations(config, cwd); repositories = parseRepositories(config); } @@ -64,8 +67,8 @@ private Map parseRepositoryHosts(JSONObject config, Path cwd) thr if (entry.value().contains("gitlab")) { var gitlab = entry.value().get("gitlab"); var uri = URIBuilder.base(gitlab.get("url").asString()).build(); - var pat = new PersonalAccessToken(gitlab.get("username").asString(), gitlab.get("pat").asString()); - ret.put(entry.name(), ForgeFactory.createGitLabHost(uri, pat)); + var pat = new Credential(gitlab.get("username").asString(), gitlab.get("pat").asString()); + ret.put(entry.name(), Forge.from("gitlab", uri, pat, gitlab.asObject())); } else if (entry.value().contains("github")) { var github = entry.value().get("github"); URI uri; @@ -74,21 +77,21 @@ private Map parseRepositoryHosts(JSONObject config, Path cwd) thr } else { uri = URIBuilder.base("https://github.com/").build(); } - Pattern webUriPattern = null; - String webUriReplacement = null; - if (github.contains("weburl")) { - webUriPattern = Pattern.compile(github.get("weburl").get("pattern").asString()); - webUriReplacement = github.get("weburl").get("replacement").asString(); - } if (github.contains("app")) { var keyFile = cwd.resolve(github.get("app").get("key").asString()); - ret.put(entry.name(), ForgeFactory.createGitHubHost(uri, webUriPattern, webUriReplacement, keyFile.toString(), - github.get("app").get("id").asString(), - github.get("app").get("installation").asString())); + try { + var keyContents = Files.readString(keyFile, StandardCharsets.UTF_8); + var pat = new Credential(github.get("app").get("id").asString() + ";" + + github.get("app").get("installation").asString(), + keyContents); + ret.put(entry.name(), Forge.from("github", uri, pat, github.asObject())); + } catch (IOException e) { + throw new ConfigurationError("Cannot find key file: " + keyFile); + } } else { - var pat = new PersonalAccessToken(github.get("username").asString(), github.get("pat").asString()); - ret.put(entry.name(), ForgeFactory.createGitHubHost(uri, pat)); + var pat = new Credential(github.get("username").asString(), github.get("pat").asString()); + ret.put(entry.name(), Forge.from("github", uri, pat, github.asObject())); } } else { throw new ConfigurationError("Host " + entry.name()); @@ -109,7 +112,11 @@ private Map parseIssueHosts(JSONObject config, Path cwd) t if (entry.value().contains("jira")) { var jira = entry.value().get("jira"); var uri = URIBuilder.base(jira.get("url").asString()).build(); - ret.put(entry.name(), IssueTrackerFactory.createJiraHost(uri, null)); + Credential credential = null; + if (jira.contains("username")) { + credential = new Credential(jira.get("username").asString(), jira.get("password").asString()); + } + ret.put(entry.name(), IssueTracker.from("jira", uri, credential, jira.asObject())); } else { throw new ConfigurationError("Host " + entry.name()); } @@ -118,6 +125,26 @@ private Map parseIssueHosts(JSONObject config, Path cwd) t return ret; } + private Map parseContinuousIntegrations(JSONObject config, Path cwd) throws ConfigurationError { + Map ret = new HashMap<>(); + + if (!config.contains("ci")) { + return ret; + } + + for (var entry : config.get("ci").fields()) { + var url = entry.value().get("url").asString(); + var ci = ContinuousIntegration.from(URI.create(url), entry.value().asObject()); + if (ci.isPresent()) { + ret.put(entry.name(), ci.get()); + } else { + throw new ConfigurationError("No continuous integration named with url: " + url); + } + } + + return ret; + } + private Map parseRepositories(JSONObject config) throws ConfigurationError { Map ret = new HashMap<>(); @@ -131,7 +158,9 @@ private Map parseRepositories(JSONObject config) throw throw new ConfigurationError("Repository " + entry.name() + " uses undefined host '" + hostName + "'"); } var host = repositoryHosts.get(hostName); - var repo = host.repository(entry.value().get("repository").asString()); + var repo = host.repository(entry.value().get("repository").asString()).orElseThrow(() -> + new ConfigurationError("Repository " + entry.value().get("repository").asString() + " is not available at " + hostName) + ); ret.put(entry.name(), repo); } @@ -158,7 +187,9 @@ private RepositoryEntry parseRepositoryEntry(String entry) throws ConfigurationE throw new ConfigurationError("Repository entry " + entry + " uses undefined host '" + hostName + "'"); } var repositoryName = entry.substring(hostSeparatorIndex + 1); - ret.repository = host.repository(repositoryName); + ret.repository = host.repository(repositoryName).orElseThrow(() -> + new ConfigurationError("Repository " + repositoryName + " is not available at " + hostName) + ); } else { if (!repositories.containsKey(entry)) { throw new ConfigurationError("Repository " + entry + " is not defined!"); @@ -233,6 +264,14 @@ public IssueProject issueProject(String name) { } } + @Override + public ContinuousIntegration continuousIntegration(String name) { + if (continuousIntegrations.containsKey(name)) { + return continuousIntegrations.get(name); + } + throw new RuntimeException("Couldn't find continuous integration with name: " + name); + } + @Override public String repositoryRef(String name) { try { diff --git a/bot/src/test/java/org/openjdk/skara/bot/BotRunnerTests.java b/bot/src/test/java/org/openjdk/skara/bot/BotRunnerTests.java index 9f87cf93f..309bcf215 100644 --- a/bot/src/test/java/org/openjdk/skara/bot/BotRunnerTests.java +++ b/bot/src/test/java/org/openjdk/skara/bot/BotRunnerTests.java @@ -276,12 +276,11 @@ void dontDiscardDifferentBlockedItems() throws TimeoutException { } @Test - @DisabledOnOs(OS.WINDOWS) void watchdogTrigger() throws TimeoutException { var countdownLatch = new CountDownLatch(1); var item = new TestBlockedWorkItem(countdownLatch); var bot = new TestBot(item); - var runner = new BotRunner(config("{ \"runner\": { \"watchdog\": \"PT0.01S\" } }"), List.of(bot)); + var runner = new BotRunner(config("{ \"runner\": { \"watchdog\": \"PT0.01S\", \"interval\": \"PT0.001S\" } }"), List.of(bot)); var errors = new ArrayList(); var log = Logger.getLogger("org.openjdk.skara.bot"); @@ -302,9 +301,9 @@ public void close() throws SecurityException { } }); - assertThrows(TimeoutException.class, () -> runner.runOnce(Duration.ofMillis(100))); + runner.run(Duration.ofMillis(100)); assertTrue(errors.size() > 0); - assertTrue(errors.size() <= 10); + assertTrue(errors.size() <= 100); countdownLatch.countDown(); } } diff --git a/bots/bridgekeeper/build.gradle b/bots/bridgekeeper/build.gradle new file mode 100644 index 000000000..f70723e89 --- /dev/null +++ b/bots/bridgekeeper/build.gradle @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +module { + name = 'org.openjdk.skara.bots.bridgekeeper' + test { + requires 'org.junit.jupiter.api' + requires 'org.openjdk.skara.vcs' + requires 'org.openjdk.skara.test' + opens 'org.openjdk.skara.bots.bridgekeeper' to 'org.junit.platform.commons' + } +} + +dependencies { + implementation project(':ci') + implementation project(':host') + implementation project(':forge') + implementation project(':issuetracker') + implementation project(':bot') + implementation project(':census') + implementation project(':json') + implementation project(':vcs') + + testImplementation project(':test') +} diff --git a/bots/bridgekeeper/src/main/java/module-info.java b/bots/bridgekeeper/src/main/java/module-info.java new file mode 100644 index 000000000..6e425b9aa --- /dev/null +++ b/bots/bridgekeeper/src/main/java/module-info.java @@ -0,0 +1,30 @@ +import org.openjdk.skara.bots.bridgekeeper.BridgekeeperBotFactory; + +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +module org.openjdk.skara.bots.bridgekeeper { + requires org.openjdk.skara.bot; + requires java.logging; + + provides org.openjdk.skara.bot.BotFactory with BridgekeeperBotFactory; +} diff --git a/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotFactory.java b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotFactory.java new file mode 100644 index 000000000..9be60c5e6 --- /dev/null +++ b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/BridgekeeperBotFactory.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.bridgekeeper; + +import org.openjdk.skara.bot.*; + +import java.time.Duration; +import java.util.*; + +public class BridgekeeperBotFactory implements BotFactory { + @Override + public String name() { + return "bridgekeeper"; + } + + @Override + public List create(BotConfiguration configuration) { + var ret = new ArrayList(); + var specific = configuration.specific(); + + for (var repo : specific.get("mirrors").asArray()) { + var bot = new PullRequestCloserBot(configuration.repository(repo.asString())); + ret.add(bot); + } + for (var repo : specific.get("pruned").fields()) { + var maxAge = Duration.parse(repo.value().get("maxage").asString()); + var bot = new PullRequestPrunerBot(configuration.repository(repo.name()), maxAge); + ret.add(bot); + } + return ret; + } +} diff --git a/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestCloserBot.java b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestCloserBot.java new file mode 100644 index 000000000..67f0fa759 --- /dev/null +++ b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestCloserBot.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.bridgekeeper; + +import org.openjdk.skara.bot.*; +import org.openjdk.skara.forge.*; + +import java.nio.file.Path; +import java.util.*; +import java.util.function.Consumer; +import java.util.logging.Logger; + +class PullRequestCloserBotWorkItem implements WorkItem { + private final Logger log = Logger.getLogger("org.openjdk.skara.bots");; + private final HostedRepository repository; + private final PullRequest pr; + private final Consumer errorHandler; + + PullRequestCloserBotWorkItem(HostedRepository repository, PullRequest pr, Consumer errorHandler) { + this.pr = pr; + this.repository = repository; + this.errorHandler = errorHandler; + } + + private final String welcomeMarker = ""; + + private void checkWelcomeMessage() { + log.info("Checking welcome message of " + pr); + + var comments = pr.comments(); + var welcomePosted = comments.stream() + .anyMatch(comment -> comment.body().contains(welcomeMarker)); + + if (!welcomePosted) { + var message = "Welcome to the OpenJDK organization on GitHub!\n\n" + + "This repository is currently a read-only git mirror of the official Mercurial " + + "repository (located at https://hg.openjdk.java.net/). As such, we are not " + + "currently accepting pull requests here. If you would like to contribute to " + + "the OpenJDK project, please see http://openjdk.java.net/contribute/ on how " + + "to proceed.\n\n" + + "This pull request will be automatically closed."; + + log.fine("Posting welcome message"); + pr.addComment(welcomeMarker + "\n\n" + message); + } + pr.setState(PullRequest.State.CLOSED); + } + + + @Override + public boolean concurrentWith(WorkItem other) { + if (!(other instanceof PullRequestCloserBotWorkItem)) { + return true; + } + PullRequestCloserBotWorkItem otherItem = (PullRequestCloserBotWorkItem)other; + if (!pr.id().equals(otherItem.pr.id())) { + return true; + } + if (!repository.name().equals(otherItem.repository.name())) { + return true; + } + return false; + } + + @Override + public void run(Path scratchPath) { + checkWelcomeMessage(); + } + + @Override + public void handleRuntimeException(RuntimeException e) { + errorHandler.accept(e); + } +} + +public class PullRequestCloserBot implements Bot { + private final HostedRepository remoteRepo; + private final PullRequestUpdateCache updateCache; + + PullRequestCloserBot(HostedRepository repo) { + this.remoteRepo = repo; + this.updateCache = new PullRequestUpdateCache(); + } + + @Override + public List getPeriodicItems() { + List ret = new LinkedList<>(); + + for (var pr : remoteRepo.pullRequests()) { + if (updateCache.needsUpdate(pr)) { + var item = new PullRequestCloserBotWorkItem(remoteRepo, pr, e -> updateCache.invalidate(pr)); + ret.add(item); + } + } + + return ret; + } +} diff --git a/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBot.java b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBot.java new file mode 100644 index 000000000..fbe3862f1 --- /dev/null +++ b/bots/bridgekeeper/src/main/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBot.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.bridgekeeper; + +import org.openjdk.skara.bot.*; +import org.openjdk.skara.forge.*; + +import java.nio.file.Path; +import java.time.*; +import java.util.*; +import java.util.logging.Logger; + +class PullRequestPrunerBotWorkItem implements WorkItem { + private final Logger log = Logger.getLogger("org.openjdk.skara.bots");; + private final HostedRepository repository; + private final PullRequest pr; + private final Duration maxAge; + + PullRequestPrunerBotWorkItem(HostedRepository repository, PullRequest pr, Duration maxAge) { + this.pr = pr; + this.repository = repository; + this.maxAge = maxAge; + } + + @Override + public boolean concurrentWith(WorkItem other) { + if (!(other instanceof PullRequestPrunerBotWorkItem)) { + return true; + } + PullRequestPrunerBotWorkItem otherItem = (PullRequestPrunerBotWorkItem) other; + if (!pr.id().equals(otherItem.pr.id())) { + return true; + } + if (!repository.name().equals(otherItem.repository.name())) { + return true; + } + return false; + } + + // Prune durations are on the order of days and weeks + private String formatDuration(Duration duration) { + var count = duration.toDays(); + var unit = "day"; + + if (count > 14) { + count /= 7; + unit = "week"; + } + if (count != 1) { + unit += "s"; + } + return count + " " + unit; + } + + private final String noticeMarker = ""; + + @Override + public void run(Path scratchPath) { + var comments = pr.comments(); + if (comments.size() > 0) { + var lastComment = comments.get(comments.size() - 1); + if (lastComment.author().equals(repository.forge().currentUser()) && lastComment.body().contains(noticeMarker)) { + var message = "@" + pr.author().userName() + " This pull request has been inactive for more than " + + formatDuration(maxAge.multipliedBy(2)) + " and will now be automatically closed. If you would " + + "like to continue working on this pull request in the future, feel free to reopen it!"; + log.fine("Posting prune message"); + pr.addComment(message); + pr.setState(PullRequest.State.CLOSED); + return; + } + } + + var message = "@" + pr.author().userName() + " This pull request has been inactive for more than " + + formatDuration(maxAge) + " and will be automatically closed if another " + formatDuration(maxAge) + + " passes without any activity. To avoid this, simply add a new comment to the pull request. Feel free " + + "to ask for assistance if you need help with progressing this pull request towards integration!"; + + log.fine("Posting prune notification message"); + pr.addComment(noticeMarker + "\n\n" + message); + } +} + +public class PullRequestPrunerBot implements Bot { + private final HostedRepository repository; + private final Duration maxAge; + + PullRequestPrunerBot(HostedRepository repository, Duration maxAge) { + this.repository = repository; + this.maxAge = maxAge; + } + + @Override + public List getPeriodicItems() { + List ret = new LinkedList<>(); + var oldestAllowed = ZonedDateTime.now().minus(maxAge); + + for (var pr : repository.pullRequests()) { + if (pr.updatedAt().isBefore(oldestAllowed)) { + var item = new PullRequestPrunerBotWorkItem(repository, pr, maxAge); + ret.add(item); + } + } + + return ret; + } +} diff --git a/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestCloserBotTests.java b/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestCloserBotTests.java new file mode 100644 index 000000000..82e5ca8f9 --- /dev/null +++ b/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestCloserBotTests.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.bridgekeeper; + +import org.openjdk.skara.issuetracker.Issue; +import org.openjdk.skara.test.*; + +import org.junit.jupiter.api.*; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class PullRequestCloserBotTests { + @Test + void simple(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var bot = new PullRequestCloserBot(author); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Let the bot see it + TestBotRunner.runPeriodicItems(bot); + + // There should now be no open PRs + var prs = author.pullRequests(); + assertEquals(0, prs.size()); + } + } + + @Test + void keepClosing(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var bot = new PullRequestCloserBot(author); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Let the bot see it + TestBotRunner.runPeriodicItems(bot); + + // There should now be no open PRs + var prs = author.pullRequests(); + assertEquals(0, prs.size()); + + // The author is persistent + pr.setState(Issue.State.OPEN); + prs = author.pullRequests(); + assertEquals(1, prs.size()); + + // But so is the bot + TestBotRunner.runPeriodicItems(bot); + prs = author.pullRequests(); + assertEquals(0, prs.size()); + + // There should still only be one welcome comment + assertEquals(1, pr.comments().size()); + } + } +} diff --git a/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBotTests.java b/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBotTests.java new file mode 100644 index 000000000..708cdf464 --- /dev/null +++ b/bots/bridgekeeper/src/test/java/org/openjdk/skara/bots/bridgekeeper/PullRequestPrunerBotTests.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.bridgekeeper; + +import org.openjdk.skara.test.*; + +import org.junit.jupiter.api.*; + +import java.io.IOException; +import java.time.Duration; + +import static org.junit.jupiter.api.Assertions.*; + +class PullRequestPrunerBotTests { + @Test + void close(TestInfo testInfo) throws IOException, InterruptedException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var bot = new PullRequestPrunerBot(author, Duration.ofMillis(1)); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Make sure the timeout expires + Thread.sleep(100); + + // Let the bot see it - it should give a notice + TestBotRunner.runPeriodicItems(bot); + + assertEquals(1, pr.comments().size()); + assertTrue(pr.comments().get(0).body().contains("will be automatically closed if")); + + pr.addComment("I'm still working on it!"); + + // Make sure the timeout expires again + Thread.sleep(100); + + // Let the bot see it - it should post a second notice + TestBotRunner.runPeriodicItems(bot); + + assertEquals(3, pr.comments().size()); + assertTrue(pr.comments().get(2).body().contains("will be automatically closed if")); + + // Make sure the timeout expires again + Thread.sleep(100); + + // The bot should now close it + TestBotRunner.runPeriodicItems(bot); + + // There should now be no open PRs + var prs = author.pullRequests(); + assertEquals(0, prs.size()); + } + } + + @Test + void dontClose(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var bot = new PullRequestPrunerBot(author, Duration.ofDays(3)); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Let the bot see it + TestBotRunner.runPeriodicItems(bot); + + // There should still be an open PR + var prs = author.pullRequests(); + assertEquals(1, prs.size()); + } + } +} diff --git a/bots/cli/build.gradle b/bots/cli/build.gradle index 71c3be055..3d154e46f 100644 --- a/bots/cli/build.gradle +++ b/bots/cli/build.gradle @@ -44,8 +44,11 @@ dependencies { implementation project(':bots:mlbridge') implementation project(':bots:mirror') implementation project(':bots:topological') + implementation project(':bots:tester') implementation project(':bots:submit') implementation project(':bots:forward') + implementation project(':bots:bridgekeeper') + implementation project(':ci') implementation project(':vcs') implementation project(':jcheck') implementation project(':host') @@ -64,7 +67,7 @@ dependencies { } images { - linux { + linux_x64 { modules = ['jdk.crypto.ec', 'org.openjdk.skara.bots.pr', 'org.openjdk.skara.bots.hgbridge', @@ -74,14 +77,16 @@ images { 'org.openjdk.skara.bots.mlbridge', 'org.openjdk.skara.bots.mirror', 'org.openjdk.skara.bots.submit', + 'org.openjdk.skara.bots.tester', 'org.openjdk.skara.bots.topological', - 'org.openjdk.skara.bots.forward'] + 'org.openjdk.skara.bots.forward', + 'org.openjdk.skara.bots.bridgekeeper'] launchers = ['skara-bots': 'org.openjdk.skara.bots.cli/org.openjdk.skara.bots.cli.BotLauncher'] options = ["--module-path", "plugins"] bundles = ['zip', 'tar.gz'] jdk { - url = 'https://download.java.net/java/GA/jdk12/GPL/openjdk-12_linux-x64_bin.tar.gz' - sha256 = 'b43bc15f4934f6d321170419f2c24451486bc848a2179af5e49d10721438dd56' + url = 'https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_linux-x64_bin.tar.gz' + sha256 = '2e01716546395694d3fad54c9b36d1cd46c5894c06f72d156772efbcf4b41335' } } } diff --git a/bots/cli/src/main/java/org/openjdk/skara/bots/cli/BotLogstashHandler.java b/bots/cli/src/main/java/org/openjdk/skara/bots/cli/BotLogstashHandler.java index 2e41d7b14..2b49d729e 100644 --- a/bots/cli/src/main/java/org/openjdk/skara/bots/cli/BotLogstashHandler.java +++ b/bots/cli/src/main/java/org/openjdk/skara/bots/cli/BotLogstashHandler.java @@ -88,7 +88,7 @@ private void publishToLogstash(Instant time, Level level, String message, Map Set parseCommits(Path base, List files, ValueParser va } public Converter resolve(Path scratchPath) throws IOException { - var localRepo = Repository.materialize(scratchPath, configurationRepo.url(), configurationRef); + var localRepo = Repository.materialize(scratchPath, configurationRepo.url(), + "+" + configurationRef + ":hgbridge_config_" + configurationRepo.name()); var replacements = parseMap(localRepo.root(), replacementsFile, field -> new Hash(field.name()), diff --git a/bots/hgbridge/src/main/java/org/openjdk/skara/bots/hgbridge/JBridgeBot.java b/bots/hgbridge/src/main/java/org/openjdk/skara/bots/hgbridge/JBridgeBot.java index e989fc887..f4c2edbc1 100644 --- a/bots/hgbridge/src/main/java/org/openjdk/skara/bots/hgbridge/JBridgeBot.java +++ b/bots/hgbridge/src/main/java/org/openjdk/skara/bots/hgbridge/JBridgeBot.java @@ -63,7 +63,8 @@ public boolean concurrentWith(WorkItem other) { } private void pushMarks(Path markSource, String destName, Path markScratchPath) throws IOException { - var marksRepo = Repository.materialize(markScratchPath, exporterConfig.marksRepo().url(), exporterConfig.marksRef()); + var marksRepo = Repository.materialize(markScratchPath, exporterConfig.marksRepo().url(), + "+" + exporterConfig.marksRef() + ":hgbridge_marks"); // We should never change existing marks var markDest = markScratchPath.resolve(destName); diff --git a/bots/merge/build.gradle b/bots/merge/build.gradle index cfe2030d9..0871ad692 100644 --- a/bots/merge/build.gradle +++ b/bots/merge/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':host') implementation project(':forge') implementation project(':issuetracker') diff --git a/bots/mirror/build.gradle b/bots/mirror/build.gradle index 5341fd05b..631ef2a4e 100644 --- a/bots/mirror/build.gradle +++ b/bots/mirror/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':host') implementation project(':forge') implementation project(':issuetracker') diff --git a/bots/mlbridge/build.gradle b/bots/mlbridge/build.gradle index 059cf2369..b49668a7f 100644 --- a/bots/mlbridge/build.gradle +++ b/bots/mlbridge/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':bot') implementation project(':mailinglist') implementation project(':host') diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java index 5bf2cc949..c6e630bf1 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveMessages.java @@ -136,10 +136,15 @@ static String composeCombinedReply(Email parent, String body, PullRequestInstanc replyFooter(prInstance); } - static String reviewCommentBody(String body, Review.Verdict verdict, String user, String role) { - var result = new StringBuilder(filterComments(body)); + static String reviewCommentBody(String body) { + return filterComments(body); + } + + static String reviewVerdictBody(String body, Review.Verdict verdict, String user, String role) { + var filteredBody = filterComments(body); + var result = new StringBuilder(); if (verdict != Review.Verdict.NONE) { - if (result.length() > 0) { + if (filteredBody.length() > 0) { result.append("\n\n"); result.append(infoSeparator); result.append("\n\n"); @@ -147,7 +152,7 @@ static String reviewCommentBody(String body, Review.Verdict verdict, String user if (verdict == Review.Verdict.APPROVED) { result.append("Approved"); } else { - result.append("Disapproved"); + result.append("Changes requested"); } result.append(" by "); result.append(user); diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveWorkItem.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveWorkItem.java index 665ab08d9..1fa55e13b 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveWorkItem.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ArchiveWorkItem.java @@ -112,7 +112,8 @@ private Optional getParentPost(Comment post, List all) { private Repository materializeArchive(Path scratchPath) { try { - return Repository.materialize(scratchPath, bot.archiveRepo().url(), pr.targetRef()); + return Repository.materialize(scratchPath, bot.archiveRepo().url(), + "+" + bot.archiveRef() + ":mlbridge_archive"); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -281,6 +282,15 @@ public void run(Path scratchPath) { reviewArchive.addComment(comment); } + // Review comments + var reviews = pr.reviews(); + for (var review : reviews) { + if (ignoreComment(review.reviewer(), review.body().orElse(""))) { + continue; + } + reviewArchive.addReview(review); + } + // File specific comments var reviewComments = pr.reviewComments(); for (var reviewComment : reviewComments) { @@ -290,13 +300,12 @@ public void run(Path scratchPath) { reviewArchive.addReviewComment(reviewComment); } - // Review comments - var reviews = pr.reviews(); + // Review verdict comments for (var review : reviews) { if (ignoreComment(review.reviewer(), review.body().orElse(""))) { continue; } - reviewArchive.addReview(review); + reviewArchive.addReviewVerdict(review); } var newMails = reviewArchive.generatedEmails(); diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/CensusInstance.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/CensusInstance.java index a22a3f07e..76f8483be 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/CensusInstance.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/CensusInstance.java @@ -49,7 +49,7 @@ private CensusInstance(Census census, JCheckConfiguration configuration, Project private static Repository initialize(HostedRepository repo, String ref, Path folder) { try { - return Repository.materialize(folder, repo.url(), ref); + return Repository.materialize(folder, repo.url(), "+" + ref + ":" + "mlbridge_census_" + repo.name()); } catch (IOException e) { throw new RuntimeException("Failed to retrieve census to " + folder, e); } diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/CommentPosterWorkItem.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/CommentPosterWorkItem.java index f44accdb1..d104ee947 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/CommentPosterWorkItem.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/CommentPosterWorkItem.java @@ -65,9 +65,12 @@ public boolean concurrentWith(WorkItem other) { private void postNewMessage(Email email) { var marker = String.format(bridgedMailMarker, Base64.getEncoder().encodeToString(email.id().address().getBytes(StandardCharsets.UTF_8))); + var body = marker + "\n" + - "Mailing list message from " + email.author().toString() + "\n\n" + - email.body(); + "*Mailing list message from [" + email.author().fullName().orElse(email.author().localPart()) + + "](mailto:" + email.author().address() + ") on [" + email.sender().localPart() + + "](mailto:" + email.sender().address() + "):*\n\n" + + TextToMarkdown.escapeFormatting(email.body()); pr.addComment(body); } diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBot.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBot.java index 6f8d5592f..56083cbdf 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBot.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBot.java @@ -36,6 +36,7 @@ public class MailingListBridgeBot implements Bot { private final EmailAddress emailAddress; private final HostedRepository codeRepo; private final HostedRepository archiveRepo; + private final String archiveRef; private final HostedRepository censusRepo; private final String censusRef; private final EmailAddress listAddress; @@ -51,7 +52,7 @@ public class MailingListBridgeBot implements Bot { private final PullRequestUpdateCache updateCache; private final Duration sendInterval; - MailingListBridgeBot(EmailAddress from, HostedRepository repo, HostedRepository archive, + MailingListBridgeBot(EmailAddress from, HostedRepository repo, HostedRepository archive, String archiveRef, HostedRepository censusRepo, String censusRef, EmailAddress list, Set ignoredUsers, Set ignoredComments, URI listArchive, String smtpServer, HostedRepository webrevStorageRepository, String webrevStorageRef, @@ -61,6 +62,7 @@ public class MailingListBridgeBot implements Bot { emailAddress = from; codeRepo = repo; archiveRepo = archive; + this.archiveRef = archiveRef; this.censusRepo = censusRepo; this.censusRef = censusRef; listAddress = list; @@ -87,6 +89,10 @@ HostedRepository archiveRepo() { return archiveRepo; } + String archiveRef() { + return archiveRef; + } + HostedRepository censusRepo() { return censusRepo; } diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotFactory.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotFactory.java index 259ce188d..4fc82bdc4 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotFactory.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotFactory.java @@ -63,6 +63,7 @@ public List create(BotConfiguration configuration) { var webrevWeb = specific.get("webrevs").get("web").asString(); var archiveRepo = configuration.repository(specific.get("archive").asString()); + var archiveRef = configuration.repositoryRef(specific.get("archive").asString()); var issueTracker = URIBuilder.base(specific.get("issues").asString()).build(); var allListNames = new HashSet(); @@ -88,7 +89,7 @@ public List create(BotConfiguration configuration) { var list = EmailAddress.parse(repoConfig.get("list").asString()); var folder = repoConfig.contains("folder") ? repoConfig.get("folder").asString() : configuration.repositoryName(repo); - var bot = new MailingListBridgeBot(from, configuration.repository(repo), archiveRepo, + var bot = new MailingListBridgeBot(from, configuration.repository(repo), archiveRepo, archiveRef, censusRepo, censusRef, list, ignoredUsers, ignoredComments, listArchive, listSmtp, webrevRepo, webrevRef, Path.of(folder), diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/PullRequestInstance.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/PullRequestInstance.java index 611df3ef2..bf7802201 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/PullRequestInstance.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/PullRequestInstance.java @@ -50,7 +50,8 @@ class PullRequestInstance { // Materialize the PR's target ref try { var repository = pr.repository(); - localRepo = Repository.materialize(localRepoPath, repository.url(), pr.targetRef()); + localRepo = Repository.materialize(localRepoPath, repository.url(), + "+" + pr.targetRef() + ":mlbridge_prinstance_" + repository.name()); targetHash = localRepo.fetch(repository.url(), pr.targetRef()); headHash = localRepo.fetch(repository.url(), pr.headHash().hex()); baseHash = localRepo.mergeBase(targetHash, headHash); diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java index 0e83a5338..d8ef5bb8b 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/ReviewArchive.java @@ -21,6 +21,7 @@ class ReviewArchive { private final Map existingIds = new HashMap<>(); private final List generated = new ArrayList<>(); private final Map generatedIds = new HashMap<>(); + private final Set approvalIds = new HashSet<>(); private final List reportedHeads; private final List reportedBases; @@ -318,7 +319,7 @@ private String projectRole(Contributor contributor) { } else if (censusInstance.project().isAuthor(contributor.username(), version)) { return "Author"; } - return "none"; + return "no project role"; } void addReview(Review review) { @@ -327,6 +328,21 @@ void addReview(Review review) { return; } + // Default parent and subject + var parent = topCommentForHash(review.hash()); + var subject = parent.subject(); + + var replyBody = ArchiveMessages.reviewCommentBody(review.body().orElse("")); + + addReplyCommon(parent, review.reviewer(), subject, replyBody, id); + } + + void addReviewVerdict(Review review) { + var id = getMessageId(review); + if (existingIds.containsKey(getStableMessageId(id))) { + return; + } + var contributor = censusInstance.namespace().get(review.reviewer().id()); var isReviewer = contributor != null && censusInstance.project().isReviewer(contributor.username(), censusInstance.configuration().census().version()); @@ -336,13 +352,12 @@ void addReview(Review review) { // Approvals by Reviewers get special treatment - post these as top-level comments if (review.verdict() == Review.Verdict.APPROVED && isReviewer) { - parent = topEmail(); - subject = "Re: [Approved] " + "RFR: " + prInstance.pr().title(); + approvalIds.add(id); } var userName = contributor != null ? contributor.username() : review.reviewer().userName() + "@" + censusInstance.namespace().name(); - var userRole = contributor != null ? projectRole(contributor) : "none"; - var replyBody = ArchiveMessages.reviewCommentBody(review.body().orElse(""), review.verdict(), userName, userRole); + var userRole = contributor != null ? projectRole(contributor) : "no OpenJDK username"; + var replyBody = ArchiveMessages.reviewVerdictBody(review.body().orElse(""), review.verdict(), userName, userRole); addReplyCommon(parent, review.reviewer(), subject, replyBody, id); } @@ -358,13 +373,16 @@ void addReviewComment(ReviewComment reviewComment) { // Add some context to the first post if (reviewComment.parent().isEmpty()) { - var contents = prInstance.pr().repository().fileContents(reviewComment.path(), reviewComment.hash().hex()).lines().collect(Collectors.toList()); - body.append(reviewComment.path()).append(" line ").append(reviewComment.line()).append(":\n\n"); - for (int i = Math.max(0, reviewComment.line() - 2); i < Math.min(contents.size(), reviewComment.line() + 1); ++i) { - body.append("> ").append(i + 1).append(": ").append(contents.get(i)).append("\n"); + try { + var contents = prInstance.pr().repository().fileContents(reviewComment.path(), reviewComment.hash().hex()).lines().collect(Collectors.toList()); + for (int i = Math.max(0, reviewComment.line() - 2); i < Math.min(contents.size(), reviewComment.line() + 1); ++i) { + body.append("> ").append(i + 1).append(": ").append(contents.get(i)).append("\n"); + } + body.append("\n"); + } catch (RuntimeException e) { + body.append("> (failed to retrieve contents of file, check the PR for context)\n"); } - body.append("\n"); } body.append(reviewComment.body()); @@ -372,6 +390,20 @@ void addReviewComment(ReviewComment reviewComment) { } List generatedEmails() { - return generated; + var finalEmails = new ArrayList(); + for (var email : generated) { + for (var approvalId : approvalIds) { + var collapsed = email.hasHeader("PR-Collapsed-IDs") ? email.headerValue("PR-Collapsed-IDs") + " " : ""; + if (email.id().equals(approvalId) || collapsed.contains(getStableMessageId(approvalId))) { + email = Email.reparent(topEmail(), email) + .subject("Re: [Approved] " + "RFR: " + prInstance.pr().title()) + .build(); + break; + } + } + finalEmails.add(email); + } + + return finalEmails; } } diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/TextToMarkdown.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/TextToMarkdown.java new file mode 100644 index 000000000..738f0d3e3 --- /dev/null +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/TextToMarkdown.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.mlbridge; + +import java.util.regex.Pattern; + +public class TextToMarkdown { + private static final Pattern punctuationPattern = Pattern.compile("([!\"#$%&'()*+,\\-./:;<=?@\\[\\]^_`{|}~])", Pattern.MULTILINE); + private static final Pattern indentedPattern = Pattern.compile("^ {4}", Pattern.MULTILINE); + + private static String escapeBackslashes(String text) { + return text.replace("\\", "\\\\"); + } + + private static String escapePunctuation(String text) { + var punctuationMatcher = punctuationPattern.matcher(text); + return punctuationMatcher.replaceAll(mr -> "\\\\" + mr.group(1)); + } + + private static String escapeIndention(String text) { + var indentedMatcher = indentedPattern.matcher(text); + return indentedMatcher.replaceAll(" "); + } + + static String escapeFormatting(String text) { + return escapeIndention(escapePunctuation(escapeBackslashes(text))); + } +} diff --git a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/WebrevStorage.java b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/WebrevStorage.java index 1c95cd188..a7ff118b8 100644 --- a/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/WebrevStorage.java +++ b/bots/mlbridge/src/main/java/org/openjdk/skara/bots/mlbridge/WebrevStorage.java @@ -107,7 +107,8 @@ private static void clearDirectory(Path directory) { URI createAndArchive(PullRequestInstance prInstance, Path scratchPath, Hash base, Hash head, String identifier) { try { - var localStorage = Repository.materialize(scratchPath, storage.url(), storageRef); + var localStorage = Repository.materialize(scratchPath, storage.url(), + "+" + storageRef + ":mlbridge_webrevs"); var relativeFolder = baseFolder.resolve(String.format("%s/webrev.%s", prInstance.id(), identifier)); var outputFolder = scratchPath.resolve(relativeFolder); // If a previous operation was interrupted there may be content here already - overwrite if so diff --git a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListArchiveReaderBotTests.java b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListArchiveReaderBotTests.java index 445bb9949..86d412871 100644 --- a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListArchiveReaderBotTests.java +++ b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListArchiveReaderBotTests.java @@ -35,7 +35,7 @@ import java.time.Duration; import java.util.*; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.*; class MailingListArchiveReaderBotTests { private void addReply(Conversation conversation, MailingList mailingList, PullRequest pr) { @@ -43,7 +43,7 @@ private void addReply(Conversation conversation, MailingList mailingList, PullRe var reply = "Looks good"; var references = first.id().toString(); - var email = Email.create(EmailAddress.from("Commenter", ""), "Re: RFR: " + pr.title(), reply) + var email = Email.create(EmailAddress.from("Commenter", "c@test.test"), "Re: RFR: " + pr.title(), reply) .recipient(first.author()) .id(EmailAddress.from(UUID.randomUUID() + "@id.id")) .header("In-Reply-To", first.id().toString()) @@ -64,7 +64,8 @@ void simpleArchive(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addAuthor(author.forge().currentUser().id()); var from = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(ignored.forge().currentUser().userName()), Set.of(), @@ -115,6 +116,9 @@ void simpleArchive(TestInfo testInfo) throws IOException { // The bridge should now have processed the reply var updated = pr.comments(); assertEquals(2, updated.size()); + assertTrue(updated.get(1).body().contains("Mailing list message from")); + assertTrue(updated.get(1).body().contains("[Commenter](mailto:c@test.test)")); + assertTrue(updated.get(1).body().contains("[test](mailto:test@" + listAddress.domain() + ")")); } } @@ -130,7 +134,8 @@ void rememberBridged(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addAuthor(author.forge().currentUser().id()); var from = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(ignored.forge().currentUser().userName()), Set.of(), diff --git a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java index a876f548b..bb0b30b6a 100644 --- a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java +++ b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/MailingListBridgeBotTests.java @@ -112,7 +112,8 @@ void simpleArchive(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addAuthor(author.forge().currentUser().id()); var from = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", listAddress, + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(ignored.forge().currentUser().userName()), Set.of(), listServer.getArchive(), listServer.getSMTP(), @@ -197,7 +198,7 @@ void simpleArchive(TestInfo testInfo) throws IOException { assertEquals("RFR: 1234: This is a pull request", mail.subject()); assertEquals(pr.author().fullName(), mail.author().fullName().orElseThrow()); assertEquals(noreplyAddress(archive), mail.author().address()); - assertEquals(from, mail.sender()); + assertEquals(listAddress, mail.sender()); assertEquals("val1", mail.headerValue("Extra1")); assertEquals("val2", mail.headerValue("Extra2")); @@ -250,7 +251,7 @@ void simpleArchive(TestInfo testInfo) throws IOException { assertEquals(3, conversations.get(0).allMessages().size()); for (var newMail : conversations.get(0).allMessages()) { assertEquals(noreplyAddress(archive), newMail.author().address()); - assertEquals(from, newMail.sender()); + assertEquals(listAddress, newMail.sender()); } assertTrue(conversations.get(0).allMessages().get(2).body().contains("This is a comment 😄")); } @@ -269,7 +270,8 @@ void reviewComment(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addAuthor(author.forge().currentUser().id()); var from = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", listAddress, + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(ignored.forge().currentUser().userName()), Set.of(), listServer.getArchive(), listServer.getSMTP(), @@ -340,7 +342,7 @@ void reviewComment(TestInfo testInfo) throws IOException { assertEquals(3, conversations.get(0).allMessages().size()); for (var newMail : conversations.get(0).allMessages()) { assertEquals(noreplyAddress(archive), newMail.author().address()); - assertEquals(from, newMail.sender()); + assertEquals(listAddress, newMail.sender()); } } } @@ -357,7 +359,8 @@ void combineComments(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addAuthor(author.forge().currentUser().id()); var from = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(), Set.of(), listServer.getArchive(), listServer.getSMTP(), @@ -445,7 +448,8 @@ void commentThreading(TestInfo testInfo) throws IOException { .addReviewer(reviewer.forge().currentUser().id()) .addAuthor(author.forge().currentUser().id()); var from = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(), Set.of(), listServer.getArchive(), listServer.getSMTP(), @@ -491,6 +495,7 @@ void commentThreading(TestInfo testInfo) throws IOException { // Finally some approvals and another comment pr.addReview(Review.Verdict.APPROVED, "Nice"); + reviewPr.addReviewComment(masterHash, editHash, reviewFile.toString(), 2, "The final review comment"); reviewPr.addReview(Review.Verdict.APPROVED, "Looks fine"); reviewPr.addReviewCommentReply(comment2, "You are welcome"); TestBotRunner.runPeriodicItems(mlBot); @@ -502,9 +507,9 @@ void commentThreading(TestInfo testInfo) throws IOException { Repository.materialize(archiveFolder.path(), archive.url(), "master"); assertEquals(9, archiveContainsCount(archiveFolder.path(), "^On.*wrote:")); - // File specific comments should appear before the approval + // File specific comments should appear after the approval var archiveText = archiveContents(archiveFolder.path()).orElseThrow(); - assertTrue(archiveText.indexOf("Looks fine") > archiveText.indexOf("You are welcome")); + assertTrue(archiveText.indexOf("Looks fine") < archiveText.indexOf("The final review comment")); // Check the mailing list var mailmanServer = MailingListServerFactory.createMailmanServer(listServer.getArchive(), listServer.getSMTP(), Duration.ZERO); @@ -543,10 +548,13 @@ void commentThreading(TestInfo testInfo) throws IOException { assertTrue(thread2reply2.body().contains("Thanks")); var replies = conversations.get(0).replies(mail); - var thread3 = conversations.get(0).replies(mail).get(2); + var thread3 = replies.get(2); assertEquals("Re: RFR: This is a pull request", thread3.subject()); - var thread4 = conversations.get(0).replies(mail).get(3); + var thread4 = replies.get(3); assertEquals("Re: [Approved] RFR: This is a pull request", thread4.subject()); + assertTrue(thread4.body().contains("Looks fine")); + assertTrue(thread4.body().contains("The final review comment")); + assertTrue(thread4.body().contains("Approved by integrationreviewer1 (Reviewer)")); } } @@ -562,7 +570,8 @@ void reviewContext(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addAuthor(author.forge().currentUser().id()); var from = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(), Set.of(), listServer.getArchive(), listServer.getSMTP(), @@ -613,7 +622,8 @@ void multipleReviewContexts(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addAuthor(author.forge().currentUser().id()); var from = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(), Set.of(), listServer.getArchive(), listServer.getSMTP(), @@ -683,7 +693,8 @@ void filterComments(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addAuthor(author.forge().currentUser().id()); var from = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(), Set.of(), listServer.getArchive(), listServer.getSMTP(), archive, "webrev", Path.of("test"), @@ -742,7 +753,8 @@ void incrementalChanges(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addAuthor(author.forge().currentUser().id()); var from = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(), Set.of(), listServer.getArchive(), listServer.getSMTP(), archive, "webrev", Path.of("test"), @@ -813,7 +825,7 @@ void incrementalChanges(TestInfo testInfo) throws IOException { assertEquals(1, conversations.size()); for (var newMail : conversations.get(0).allMessages()) { assertEquals(noreplyAddress(archive), newMail.author().address()); - assertEquals(from, newMail.sender()); + assertEquals(listAddress, newMail.sender()); } // Add a comment @@ -863,7 +875,8 @@ void rebased(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addAuthor(author.forge().currentUser().id()); var sender = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(sender, author, archive, censusBuilder.build(), "master", + var mlBot = new MailingListBridgeBot(sender, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(), Set.of(), listServer.getArchive(), listServer.getSMTP(), archive, "webrev", Path.of("test"), @@ -933,7 +946,7 @@ void rebased(TestInfo testInfo) throws IOException { assertEquals(1, conversations.size()); for (var newMail : conversations.get(0).allMessages()) { assertEquals(noreplyAddress(archive), newMail.author().address()); - assertEquals(sender, newMail.sender()); + assertEquals(listAddress, newMail.sender()); assertFalse(newMail.hasHeader("PR-Head-Hash")); } assertEquals("Re: [Rev 01] RFR: This is a pull request", conversations.get(0).allMessages().get(1).subject()); @@ -954,7 +967,8 @@ void skipAddingExistingWebrev(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addAuthor(author.forge().currentUser().id()); var from = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(ignored.forge().currentUser().userName()), Set.of(), @@ -1029,7 +1043,8 @@ void notifyReviewVerdicts(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addReviewer(reviewer.forge().currentUser().id()) .addAuthor(author.forge().currentUser().id()); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(), Set.of(), listServer.getArchive(), listServer.getSMTP(), archive, "webrev", Path.of("test"), @@ -1063,7 +1078,7 @@ void notifyReviewVerdicts(TestInfo testInfo) throws IOException { // The archive should contain a note Repository.materialize(archiveFolder.path(), archive.url(), "master"); - assertEquals(1, archiveContainsCount(archiveFolder.path(), "Disapproved by ")); + assertEquals(1, archiveContainsCount(archiveFolder.path(), "Changes requested by ")); assertEquals(1, archiveContainsCount(archiveFolder.path(), " by integrationreviewer1")); if (author.forge().supportsReviewBody()) { assertEquals(1, archiveContainsCount(archiveFolder.path(), "Reason 1")); @@ -1091,7 +1106,7 @@ void notifyReviewVerdicts(TestInfo testInfo) throws IOException { // The archive should contain another note Repository.materialize(archiveFolder.path(), archive.url(), "master"); - assertEquals(2, archiveContainsCount(archiveFolder.path(), "Disapproved by ")); + assertEquals(2, archiveContainsCount(archiveFolder.path(), "Changes requested by ")); if (author.forge().supportsReviewBody()) { assertEquals(1, archiveContainsCount(archiveFolder.path(), "Reason 3")); } @@ -1111,7 +1126,8 @@ void ignoreComments(TestInfo testInfo) throws IOException { var censusBuilder = credentials.getCensusBuilder() .addAuthor(author.forge().currentUser().id()); var from = EmailAddress.from("test", "test@test.mail"); - var mlBot = new MailingListBridgeBot(from, author, archive, censusBuilder.build(), "master", + var mlBot = new MailingListBridgeBot(from, author, archive, "master", + censusBuilder.build(), "master", listAddress, Set.of(ignored.forge().currentUser().userName()), Set.of(Pattern.compile("ignore this comment", Pattern.MULTILINE | Pattern.DOTALL)), diff --git a/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/TextToMarkdownTests.java b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/TextToMarkdownTests.java new file mode 100644 index 000000000..e2162694e --- /dev/null +++ b/bots/mlbridge/src/test/java/org/openjdk/skara/bots/mlbridge/TextToMarkdownTests.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.mlbridge; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class TextToMarkdownTests { + @Test + void punctuation() { + assertEquals("1\\. Not a list", TextToMarkdown.escapeFormatting("1. Not a list")); + assertEquals("\\*not emphasized\\*", TextToMarkdown.escapeFormatting("*not emphasized*")); + assertEquals("\\\\n", TextToMarkdown.escapeFormatting("\\n")); + } + + @Test + void indented() { + assertEquals(" hello", TextToMarkdown.escapeFormatting(" hello")); + } + + @Test + void preserveQuoting() { + assertEquals("> quoted", TextToMarkdown.escapeFormatting("> quoted")); + } +} diff --git a/bots/notify/build.gradle b/bots/notify/build.gradle index 7c8483b3c..adebeab41 100644 --- a/bots/notify/build.gradle +++ b/bots/notify/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':host') implementation project(':network') implementation project(':bot') diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/CommitFormatters.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/CommitFormatters.java new file mode 100644 index 000000000..ed119b6c0 --- /dev/null +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/CommitFormatters.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.notify; + +import org.openjdk.skara.forge.HostedRepository; +import org.openjdk.skara.vcs.*; + +import java.io.*; +import java.time.format.DateTimeFormatter; + +class CommitFormatters { + static String toTextBrief(HostedRepository repository, Commit commit) { + var writer = new StringWriter(); + var printer = new PrintWriter(writer); + + printer.println("Changeset: " + commit.hash().abbreviate()); + printer.println("Author: " + commit.author().name() + " <" + commit.author().email() + ">"); + if (!commit.author().equals(commit.committer())) { + printer.println("Committer: " + commit.committer().name() + " <" + commit.committer().email() + ">"); + } + printer.println("Date: " + commit.date().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss +0000"))); + printer.println("URL: " + repository.webUrl(commit.hash())); + + return writer.toString(); + } + + private static String patchToText(Patch patch) { + if (patch.status().isAdded()) { + return "+ " + patch.target().path().orElseThrow(); + } else if (patch.status().isDeleted()) { + return "- " + patch.source().path().orElseThrow(); + } else if (patch.status().isModified()) { + return "! " + patch.target().path().orElseThrow(); + } else { + return "= " + patch.target().path().orElseThrow(); + } + } + + static String toText(HostedRepository repository, Commit commit) { + var writer = new StringWriter(); + var printer = new PrintWriter(writer); + + printer.print(toTextBrief(repository, commit)); + printer.println(); + printer.println(String.join("\n", commit.message())); + printer.println(); + + for (var diff : commit.parentDiffs()) { + for (var patch : diff.patches()) { + printer.println(patchToText(patch)); + } + } + + return writer.toString(); + } +} diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/IssueUpdater.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/IssueUpdater.java new file mode 100644 index 000000000..6b622985b --- /dev/null +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/IssueUpdater.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.notify; + +import org.openjdk.skara.forge.HostedRepository; +import org.openjdk.skara.issuetracker.Issue; +import org.openjdk.skara.issuetracker.IssueProject; +import org.openjdk.skara.vcs.*; +import org.openjdk.skara.vcs.openjdk.*; + +import java.util.List; +import java.util.logging.Logger; + +public class IssueUpdater implements UpdateConsumer { + private final IssueProject issueProject; + private final Logger log = Logger.getLogger("org.openjdk.skara.bots.notify"); + + IssueUpdater(IssueProject issueProject) { + this.issueProject = issueProject; + } + + @Override + public void handleCommits(HostedRepository repository, List commits, Branch branch) { + for (var commit : commits) { + var commitNotification = CommitFormatters.toTextBrief(repository, commit); + var commitMessage = CommitMessageParsers.v1.parse(commit); + for (var commitIssue : commitMessage.issues()) { + var issue = issueProject.issue(commitIssue.id()); + if (issue.isEmpty()) { + log.severe("Cannot update issue " + commitIssue.id() + " with commit " + commit.hash().abbreviate() + + " - issue not found in issue project"); + continue; + } + issue.get().addComment(commitNotification); + issue.get().setState(Issue.State.CLOSED); + } + } + } + + @Override + public void handleOpenJDKTagCommits(HostedRepository repository, List commits, OpenJDKTag tag, Tag.Annotated annotated) { + + } + + @Override + public void handleTagCommit(HostedRepository repository, Commit commit, Tag tag, Tag.Annotated annotation) { + + } + + @Override + public void handleNewBranch(HostedRepository repository, List commits, Branch parent, Branch branch) { + + } +} diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBot.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBot.java index a2c755b0b..32b236b61 100644 --- a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBot.java +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBot.java @@ -35,7 +35,7 @@ import java.util.*; import java.util.logging.Logger; import java.util.regex.Pattern; -import java.util.stream.Collectors; +import java.util.stream.*; class JNotifyBot implements Bot, WorkItem { private final Logger log = Logger.getLogger("org.openjdk.skara.bots");; @@ -166,33 +166,55 @@ private void handleTags(Repository localRepo, UpdateHistory history) throws IOEx .map(Optional::get) .collect(Collectors.toSet()); var newJdkTags = newTags.stream() - .map(OpenJDKTag::create) - .filter(Optional::isPresent) - .map(Optional::get) - .sorted(Comparator.comparingInt(OpenJDKTag::buildNum)) - .collect(Collectors.toList()); - + .map(OpenJDKTag::create) + .filter(Optional::isPresent) + .map(Optional::get) + .sorted(Comparator.comparingInt(OpenJDKTag::buildNum)) + .collect(Collectors.toList()); for (var tag : newJdkTags) { // Update the history first - if there is a problem here we don't want to send out multiple updates history.addTags(List.of(tag.tag())); var commits = new ArrayList(); + + // Try to determine which commits are new since the last build var previous = existingPrevious(tag, allJdkTags); - if (previous.isEmpty()) { + if (previous.isPresent()) { + commits.addAll(localRepo.commits(previous.get().tag() + ".." + tag.tag()).asList()); + } + + // If none are found, just include the commit that was tagged + if (commits.isEmpty()) { var commit = localRepo.lookup(tag.tag()); if (commit.isEmpty()) { throw new RuntimeException("Failed to lookup tag '" + tag.toString() + "'"); } else { commits.add(commit.get()); - log.warning("No previous tag found for '" + tag.tag() + "'"); } - } else { - commits.addAll(localRepo.commits(previous.get().tag() + ".." + tag.tag()).asList()); } Collections.reverse(commits); + var annotation = localRepo.annotate(tag.tag()); + for (var updater : updaters) { + updater.handleOpenJDKTagCommits(repository, commits, tag, annotation.orElse(null)); + } + } + + var newNonJdkTags = newTags.stream() + .filter(tag -> OpenJDKTag.create(tag).isEmpty()) + .collect(Collectors.toList()); + for (var tag : newNonJdkTags) { + // Update the history first - if there is a problem here we don't want to send out multiple updates + history.addTags(List.of(tag)); + + var commit = localRepo.lookup(tag); + if (commit.isEmpty()) { + throw new RuntimeException("Failed to lookup tag '" + tag.toString() + "'"); + } + + var annotation = localRepo.annotate(tag); for (var updater : updaters) { - updater.handleTagCommits(repository, commits, tag); + updater.handleTagCommit(repository, commit.get(), tag, annotation.orElse(null)); } } } diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBotFactory.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBotFactory.java index f4eb8e65b..ec0f051bf 100644 --- a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBotFactory.java +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JNotifyBotFactory.java @@ -106,10 +106,15 @@ public List create(BotConfiguration configuration) { .collect(Collectors.toMap(JSONObject.Field::name, field -> field.value().asString())) : Map.of(); var author = mailinglist.contains("author") ? EmailAddress.parse(mailinglist.get("author").asString()) : null; + var allowedDomains = author == null ? Pattern.compile(mailinglist.get("domains").asString()) : null; updaters.add(new MailingListUpdater(listServer.getList(recipient), recipientAddress, sender, author, - includeBranchNames, mode, headers)); + includeBranchNames, mode, headers, allowedDomains)); } } + if (repo.value().contains("issues")) { + var issueProject = configuration.issueProject(repo.value().get("issues").asString()); + updaters.add(new IssueUpdater(issueProject)); + } if (updaters.isEmpty()) { log.warning("No consumers configured for notify bot repository: " + repoName); diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JsonUpdater.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JsonUpdater.java index 598633384..6599470c3 100644 --- a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JsonUpdater.java +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/JsonUpdater.java @@ -86,7 +86,7 @@ public void handleCommits(HostedRepository repository, List commits, Bra } @Override - public void handleTagCommits(HostedRepository repository, List commits, OpenJDKTag tag) { + public void handleOpenJDKTagCommits(HostedRepository repository, List commits, OpenJDKTag tag, Tag.Annotated annotation) { var build = String.format("b%02d", tag.buildNum()); try (var writer = new JsonUpdateWriter(path, repository.name())) { var issues = new ArrayList(); @@ -100,7 +100,10 @@ public void handleTagCommits(HostedRepository repository, List commits, } @Override - public void handleNewBranch(HostedRepository repository, List commits, Branch parent, Branch branch) { + public void handleTagCommit(HostedRepository repository, Commit commit, Tag tag, Tag.Annotated annotation) { + } + @Override + public void handleNewBranch(HostedRepository repository, List commits, Branch parent, Branch branch) { } } diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/MailingListUpdater.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/MailingListUpdater.java index db3a455a9..3df015d51 100644 --- a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/MailingListUpdater.java +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/MailingListUpdater.java @@ -32,9 +32,9 @@ import java.time.Duration; import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.logging.Logger; public class MailingListUpdater implements UpdateConsumer { private final MailingList list; @@ -44,6 +44,7 @@ public class MailingListUpdater implements UpdateConsumer { private final boolean includeBranch; private final Mode mode; private final Map headers; + private final Pattern allowedAuthorDomains; private final Logger log = Logger.getLogger("org.openjdk.skara.bots.notify"); enum Mode { @@ -53,7 +54,7 @@ enum Mode { } MailingListUpdater(MailingList list, EmailAddress recipient, EmailAddress sender, EmailAddress author, - boolean includeBranch, Mode mode, Map headers) { + boolean includeBranch, Mode mode, Map headers, Pattern allowedAuthorDomains) { this.list = list; this.recipient = recipient; this.sender = sender; @@ -61,47 +62,39 @@ enum Mode { this.includeBranch = includeBranch; this.mode = mode; this.headers = headers; + this.allowedAuthorDomains = allowedAuthorDomains; } - private String patchToText(Patch patch) { - if (patch.status().isAdded()) { - return "+ " + patch.target().path().orElseThrow(); - } else if (patch.status().isDeleted()) { - return "- " + patch.source().path().orElseThrow(); - } else if (patch.status().isModified()) { - return "! " + patch.target().path().orElseThrow(); - } else { - return "= " + patch.target().path().orElseThrow(); - } - } - - private String commitToText(HostedRepository repository, Commit commit) { + private String tagAnnotationToText(HostedRepository repository, Tag.Annotated annotation) { var writer = new StringWriter(); var printer = new PrintWriter(writer); - printer.println("Changeset: " + commit.hash().abbreviate()); - printer.println("Author: " + commit.author().name() + " <" + commit.author().email() + ">"); - if (!commit.author().equals(commit.committer())) { - printer.println("Committer: " + commit.committer().name() + " <" + commit.committer().email() + ">"); - } - printer.println("Date: " + commit.date().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss +0000"))); - printer.println("URL: " + repository.webUrl(commit.hash())); - printer.println(); - printer.println(String.join("\n", commit.message())); + printer.println("Tagged by: " + annotation.author().name() + " <" + annotation.author().email() + ">"); + printer.println("Date: " + annotation.date().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss +0000"))); printer.println(); + printer.print(String.join("\n", annotation.message())); - for (var diff : commit.parentDiffs()) { - for (var patch : diff.patches()) { - printer.println(patchToText(patch)); - } + return writer.toString(); + } + + private EmailAddress filteredAuthor(EmailAddress commitAddress) { + if (author != null) { + return author; } + var allowedAuthorMatcher = allowedAuthorDomains.matcher(commitAddress.domain()); + if (!allowedAuthorMatcher.matches()) { + return sender; + } else { + return commitAddress; + } + } - return writer.toString(); + private EmailAddress commitToAuthor(Commit commit) { + return filteredAuthor(EmailAddress.from(commit.committer().name(), commit.committer().email())); } - private EmailAddress commitsToAuthor(List commits) { - var commit = commits.get(commits.size() - 1); - return EmailAddress.from(commit.committer().name(), commit.committer().email()); + private EmailAddress annotationToAuthor(Tag.Annotated annotation) { + return filteredAuthor(EmailAddress.from(annotation.author().name(), annotation.author().email())); } private String commitsToSubject(HostedRepository repository, List commits, Branch branch) { @@ -123,12 +116,12 @@ private String commitsToSubject(HostedRepository repository, List commit return subject.toString(); } - private String tagToSubject(HostedRepository repository, Hash hash, OpenJDKTag tag) { + private String tagToSubject(HostedRepository repository, Hash hash, Tag tag) { return repository.repositoryType().shortName() + ": " + repository.name() + ": Added tag " + - tag.tag() + + tag + " for changeset " + hash.abbreviate(); } @@ -162,11 +155,11 @@ private List filterAndSendPrCommits(HostedRepository repository, List commi var printer = new PrintWriter(writer); for (var commit : commits) { - printer.println(commitToText(repository, commit)); + printer.println(CommitFormatters.toText(repository, commit)); } var subject = commitsToSubject(repository, commits, branch); - var finalAuthor = author != null ? author : commitsToAuthor(commits); + var lastCommit = commits.get(commits.size() - 1); + var commitAddress = filteredAuthor(EmailAddress.from(lastCommit.committer().name(), lastCommit.committer().email())); var email = Email.create(subject, writer.toString()) .sender(sender) - .author(finalAuthor) + .author(commitAddress) .recipient(recipient) .headers(headers) .build(); @@ -216,13 +210,19 @@ public void handleCommits(HostedRepository repository, List commits, Bra } @Override - public void handleTagCommits(HostedRepository repository, List commits, OpenJDKTag tag) { + public void handleOpenJDKTagCommits(HostedRepository repository, List commits, OpenJDKTag tag, Tag.Annotated annotation) { if (mode == Mode.PR_ONLY) { return; } var writer = new StringWriter(); var printer = new PrintWriter(writer); + var taggedCommit = commits.get(commits.size() - 1); + if (annotation != null) { + printer.println(tagAnnotationToText(repository, annotation)); + } + printer.println(CommitFormatters.toTextBrief(repository, taggedCommit)); + printer.println("The following commits are included in " + tag.tag()); printer.println("========================================================"); for (var commit : commits) { @@ -233,17 +233,47 @@ public void handleTagCommits(HostedRepository repository, List commits, printer.println(); } - var tagCommit = commits.get(commits.size() - 1); - var subject = tagToSubject(repository, tagCommit.hash(), tag); - var finalAuthor = author != null ? author : commitsToAuthor(commits); + var subject = tagToSubject(repository, taggedCommit.hash(), tag.tag()); var email = Email.create(subject, writer.toString()) .sender(sender) - .author(finalAuthor) .recipient(recipient) - .headers(headers) - .build(); + .headers(headers); - list.post(email); + if (annotation != null) { + email.author(annotationToAuthor(annotation)); + } else { + email.author(commitToAuthor(taggedCommit)); + } + + list.post(email.build()); + } + + @Override + public void handleTagCommit(HostedRepository repository, Commit commit, Tag tag, Tag.Annotated annotation) { + if (mode == Mode.PR_ONLY) { + return; + } + var writer = new StringWriter(); + var printer = new PrintWriter(writer); + + if (annotation != null) { + printer.println(tagAnnotationToText(repository, annotation)); + } + printer.println(CommitFormatters.toTextBrief(repository, commit)); + + var subject = tagToSubject(repository, commit.hash(), tag); + var email = Email.create(subject, writer.toString()) + .sender(sender) + .recipient(recipient) + .headers(headers); + + if (annotation != null) { + email.author(annotationToAuthor(annotation)); + } else { + email.author(commitToAuthor(commit)); + } + + list.post(email.build()); } private String newBranchSubject(HostedRepository repository, List commits, Branch parent, Branch branch) { @@ -271,7 +301,7 @@ public void handleNewBranch(HostedRepository repository, List commits, B var printer = new PrintWriter(writer); if (commits.size() > 0) { - printer.println("The following commits are unique to the " + branch.name() + " branch"); + printer.println("The following commits are unique to the " + branch.name() + " branch:"); printer.println("========================================================"); for (var commit : commits) { printer.print(commit.hash().abbreviate()); @@ -285,7 +315,8 @@ public void handleNewBranch(HostedRepository repository, List commits, B } var subject = newBranchSubject(repository, commits, parent, branch); - var finalAuthor = author != null ? author : commits.size() > 0 ? commitsToAuthor(commits) : sender; + var finalAuthor = commits.size() > 0 ? commitToAuthor(commits.get(commits.size() - 1)) : sender; + var email = Email.create(subject, writer.toString()) .sender(sender) .author(finalAuthor) diff --git a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/UpdateConsumer.java b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/UpdateConsumer.java index 5dac766bb..a8ece6385 100644 --- a/bots/notify/src/main/java/org/openjdk/skara/bots/notify/UpdateConsumer.java +++ b/bots/notify/src/main/java/org/openjdk/skara/bots/notify/UpdateConsumer.java @@ -30,6 +30,14 @@ public interface UpdateConsumer { void handleCommits(HostedRepository repository, List commits, Branch branch); - void handleTagCommits(HostedRepository repository, List commits, OpenJDKTag tag); + void handleOpenJDKTagCommits(HostedRepository repository, List commits, OpenJDKTag tag, Tag.Annotated annotated); + void handleTagCommit(HostedRepository repository, Commit commit, Tag tag, Tag.Annotated annotation); void handleNewBranch(HostedRepository repository, List commits, Branch parent, Branch branch); + + default void handleOpenJDKTagCommits(HostedRepository repository, List commits, OpenJDKTag tag) { + handleOpenJDKTagCommits(repository, commits, tag, null); + } + default void handleTagCommit(HostedRepository repository, Commit commit, Tag tag) { + handleTagCommit(repository, commit, tag, null); + } } diff --git a/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java b/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java index bc3a8c0c0..c6e100a93 100644 --- a/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java +++ b/bots/notify/src/test/java/org/openjdk/skara/bots/notify/UpdaterTests.java @@ -186,7 +186,7 @@ void testMailingList(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, MailingListUpdater.Mode.ALL, - Map.of("extra1", "value1", "extra2", "value2")); + Map.of("extra1", "value1", "extra2", "value2"), Pattern.compile("none")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -200,8 +200,8 @@ void testMailingList(TestInfo testInfo) throws IOException { var conversations = mailmanList.conversations(Duration.ofDays(1)); var email = conversations.get(0).first(); - assertEquals(sender, email.sender()); - assertEquals(EmailAddress.from("testauthor", "ta@none.none"), email.author()); + assertEquals(listAddress, email.sender()); + assertEquals(sender, email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertTrue(email.subject().contains(": 23456789: More fixes")); assertFalse(email.subject().contains("master")); @@ -237,7 +237,7 @@ void testMailingListMultiple(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, - MailingListUpdater.Mode.ALL, Map.of()); + MailingListUpdater.Mode.ALL, Map.of(), Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -256,7 +256,7 @@ void testMailingListMultiple(TestInfo testInfo) throws IOException { var conversations = mailmanList.conversations(Duration.ofDays(1)); var email = conversations.get(0).first(); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(EmailAddress.from("another_author", "another@author.example.com"), email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertTrue(email.subject().contains(": 2 new changesets")); @@ -290,7 +290,7 @@ void testMailingListSponsored(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, - MailingListUpdater.Mode.ALL, Map.of()); + MailingListUpdater.Mode.ALL, Map.of(), Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -306,7 +306,7 @@ void testMailingListSponsored(TestInfo testInfo) throws IOException { var conversations = mailmanList.conversations(Duration.ofDays(1)); var email = conversations.get(0).first(); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(EmailAddress.from("committer", "committer@test.test"), email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertTrue(email.body().contains("Changeset: " + editHash.abbreviate())); @@ -340,7 +340,7 @@ void testMailingListMultipleBranches(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var author = EmailAddress.from("author", "author@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, author, true, - MailingListUpdater.Mode.ALL, Map.of()); + MailingListUpdater.Mode.ALL, Map.of(), Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master|another"), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -357,7 +357,7 @@ void testMailingListMultipleBranches(TestInfo testInfo) throws IOException { var conversations = mailmanList.conversations(Duration.ofDays(1)); var email = conversations.get(0).first(); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(author, email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertFalse(email.subject().contains("another")); @@ -379,7 +379,8 @@ void testMailingListMultipleBranches(TestInfo testInfo) throws IOException { conversations = mailmanList.conversations(Duration.ofDays(1)); conversations.sort(Comparator.comparing(conversation -> conversation.first().subject())); email = conversations.get(0).first(); - assertEquals(email.sender(), sender); + assertEquals(author, email.author()); + assertEquals(listAddress, email.sender()); assertEquals(email.recipients(), List.of(listAddress)); assertTrue(email.subject().contains(": another: 456789AB: Yet more fixes")); assertFalse(email.subject().contains("master")); @@ -412,7 +413,8 @@ void testMailingListPROnly(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var author = EmailAddress.from("author", "author@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, author, false, - MailingListUpdater.Mode.PR_ONLY, Map.of("extra1", "value1")); + MailingListUpdater.Mode.PR_ONLY, Map.of("extra1", "value1"), + Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -449,7 +451,7 @@ void testMailingListPROnly(TestInfo testInfo) throws IOException { assertEquals(1, conversations.size()); var first = conversations.get(0).first(); var email = conversations.get(0).replies(first).get(0); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(author, email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertEquals("Re: [Integrated] RFR: My PR", email.subject()); @@ -489,7 +491,7 @@ void testMailingListPR(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, - MailingListUpdater.Mode.PR, Map.of()); + MailingListUpdater.Mode.PR, Map.of(), Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -536,7 +538,7 @@ void testMailingListPR(TestInfo testInfo) throws IOException { var pushConversation = conversations.get(1); var prEmail = prConversation.replies(prConversation.first()).get(0); - assertEquals(sender, prEmail.sender()); + assertEquals(listAddress, prEmail.sender()); assertEquals(EmailAddress.from("testauthor", "ta@none.none"), prEmail.author()); assertEquals(prEmail.recipients(), List.of(listAddress)); assertEquals("Re: [Integrated] RFR: My PR", prEmail.subject()); @@ -547,7 +549,7 @@ void testMailingListPR(TestInfo testInfo) throws IOException { assertFalse(prEmail.body().contains(masterHash.abbreviate())); var pushEmail = pushConversation.first(); - assertEquals(sender, pushEmail.sender()); + assertEquals(listAddress, pushEmail.sender()); assertEquals(EmailAddress.from("testauthor", "ta@none.none"), pushEmail.author()); assertEquals(pushEmail.recipients(), List.of(listAddress)); assertTrue(pushEmail.subject().contains("23456789: More fixes")); @@ -564,7 +566,7 @@ void testMailinglistTag(TestInfo testInfo) throws IOException { var localRepo = CheckableRepository.init(localRepoFolder, repo.repositoryType()); credentials.commitLock(localRepo); var masterHash = localRepo.resolve("master").orElseThrow(); - localRepo.tag(masterHash, "jdk-12+1", "Added tag 1", "Duke", "duke@openjdk.java.net"); + localRepo.tag(masterHash, "jdk-12+1", "Added tag 1", "Duke Tagger", "tagger@openjdk.java.net"); localRepo.pushAll(repo.url()); var listAddress = EmailAddress.parse(listServer.createList("test")); @@ -576,9 +578,10 @@ void testMailinglistTag(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, MailingListUpdater.Mode.ALL, - Map.of("extra1", "value1", "extra2", "value2")); + Map.of("extra1", "value1", "extra2", "value2"), + Pattern.compile(".*")); var prOnlyUpdater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, - MailingListUpdater.Mode.PR_ONLY, Map.of()); + MailingListUpdater.Mode.PR_ONLY, Map.of(), Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater, prOnlyUpdater)); @@ -588,23 +591,24 @@ void testMailinglistTag(TestInfo testInfo) throws IOException { var editHash = CheckableRepository.appendAndCommit(localRepo, "Another line", "23456789: More fixes"); localRepo.fetch(repo.url(), "history:history"); - localRepo.tag(editHash, "jdk-12+2", "Added tag 2", "Duke", "duke@openjdk.java.net"); + localRepo.tag(editHash, "jdk-12+2", "Added tag 2", "Duke Tagger", "tagger@openjdk.java.net"); CheckableRepository.appendAndCommit(localRepo, "Another line 1", "34567890: Even more fixes"); CheckableRepository.appendAndCommit(localRepo, "Another line 2", "45678901: Yet even more fixes"); var editHash2 = CheckableRepository.appendAndCommit(localRepo, "Another line 3", "56789012: Still even more fixes"); - localRepo.tag(editHash2, "jdk-12+4", "Added tag 3", "Duke", "duke@openjdk.java.net"); + localRepo.tag(editHash2, "jdk-12+4", "Added tag 3", "Duke Tagger", "tagger@openjdk.java.net"); CheckableRepository.appendAndCommit(localRepo, "Another line 4", "67890123: Brand new fixes"); var editHash3 = CheckableRepository.appendAndCommit(localRepo, "Another line 5", "78901234: More brand new fixes"); - localRepo.tag(editHash3, "jdk-13+0", "Added tag 4", "Duke", "duke@openjdk.java.net"); + localRepo.tag(editHash3, "jdk-13+0", "Added tag 4", "Duke Tagger", "tagger@openjdk.java.net"); localRepo.pushAll(repo.url()); TestBotRunner.runPeriodicItems(notifyBot); listServer.processIncoming(); listServer.processIncoming(); listServer.processIncoming(); + listServer.processIncoming(); var conversations = mailmanList.conversations(Duration.ofDays(1)); - assertEquals(3, conversations.size()); + assertEquals(4, conversations.size()); for (var conversation : conversations) { var email = conversation.first(); @@ -615,6 +619,7 @@ void testMailinglistTag(TestInfo testInfo) throws IOException { assertFalse(email.body().contains("56789012: Still even more fixes")); assertFalse(email.body().contains("67890123: Brand new fixes")); assertFalse(email.body().contains("78901234: More brand new fixes")); + assertEquals(EmailAddress.from("Duke Tagger", "tagger@openjdk.java.net"), email.author()); } else if (email.subject().equals("git: test: Added tag jdk-12+4 for changeset " + editHash2.abbreviate())) { assertFalse(email.body().contains("23456789: More fixes")); assertTrue(email.body().contains("34567890: Even more fixes")); @@ -622,6 +627,7 @@ void testMailinglistTag(TestInfo testInfo) throws IOException { assertTrue(email.body().contains("56789012: Still even more fixes")); assertFalse(email.body().contains("67890123: Brand new fixes")); assertFalse(email.body().contains("78901234: More brand new fixes")); + assertEquals(EmailAddress.from("Duke Tagger", "tagger@openjdk.java.net"), email.author()); } else if (email.subject().equals("git: test: Added tag jdk-13+0 for changeset " + editHash3.abbreviate())) { assertFalse(email.body().contains("23456789: More fixes")); assertFalse(email.body().contains("34567890: Even more fixes")); @@ -629,13 +635,15 @@ void testMailinglistTag(TestInfo testInfo) throws IOException { assertFalse(email.body().contains("56789012: Still even more fixes")); assertFalse(email.body().contains("67890123: Brand new fixes")); assertTrue(email.body().contains("78901234: More brand new fixes")); - } else if (email.subject().equals("git: test: 4 new changesets")) { + assertEquals(EmailAddress.from("Duke Tagger", "tagger@openjdk.java.net"), email.author()); + } else if (email.subject().equals("git: test: 6 new changesets")) { assertTrue(email.body().contains("23456789: More fixes")); assertTrue(email.body().contains("34567890: Even more fixes")); assertTrue(email.body().contains("45678901: Yet even more fixes")); assertTrue(email.body().contains("56789012: Still even more fixes")); assertTrue(email.body().contains("67890123: Brand new fixes")); assertTrue(email.body().contains("78901234: More brand new fixes")); + assertEquals(EmailAddress.from("testauthor", "ta@none.none"), email.author()); } else { fail("Mismatched subject: " + email.subject()); } @@ -668,7 +676,8 @@ void testMailingListBranch(TestInfo testInfo) throws IOException { var sender = EmailAddress.from("duke", "duke@duke.duke"); var updater = new MailingListUpdater(mailmanList, listAddress, sender, null, false, MailingListUpdater.Mode.ALL, - Map.of("extra1", "value1", "extra2", "value2")); + Map.of("extra1", "value1", "extra2", "value2"), + Pattern.compile(".*")); var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master|newbranch."), tagStorage, branchStorage, List.of(updater)); // No mail should be sent on the first run as there is no history @@ -683,7 +692,7 @@ void testMailingListBranch(TestInfo testInfo) throws IOException { var conversations = mailmanList.conversations(Duration.ofDays(1)); var email = conversations.get(0).first(); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(EmailAddress.from("testauthor", "ta@none.none"), email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertEquals("git: test: created branch newbranch1 based on the branch master containing 2 unique commits", email.subject()); @@ -704,11 +713,49 @@ void testMailingListBranch(TestInfo testInfo) throws IOException { .filter(c -> !c.equals(conversations.get(0))) .findFirst().orElseThrow(); email = newConversation.first(); - assertEquals(sender, email.sender()); + assertEquals(listAddress, email.sender()); assertEquals(sender, email.author()); assertEquals(email.recipients(), List.of(listAddress)); assertEquals("git: test: created branch newbranch2 based on the branch newbranch1 containing 0 unique commits", email.subject()); assertEquals("The new branch newbranch2 is currently identical to the newbranch1 branch.", email.body()); } } + + @Test + void testIssue(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var repo = credentials.getHostedRepository(); + var repoFolder = tempFolder.path().resolve("repo"); + var localRepo = CheckableRepository.init(repoFolder, repo.repositoryType()); + credentials.commitLock(localRepo); + localRepo.pushAll(repo.url()); + + var tagStorage = createTagStorage(repo); + var branchStorage = createBranchStorage(repo); + var storageFolder = tempFolder.path().resolve("storage"); + + var issueProject = credentials.getIssueProject(); + var updater = new IssueUpdater(issueProject); + var notifyBot = new JNotifyBot(repo, storageFolder, Pattern.compile("master"), tagStorage, branchStorage, List.of(updater)); + + // Initialize history + TestBotRunner.runPeriodicItems(notifyBot); + + // Create an issue and commit a fix + var issue = issueProject.createIssue("This is an issue", List.of("Indeed")); + var editHash = CheckableRepository.appendAndCommit(localRepo, "Another line", issue.id() + ": Fix that issue"); + localRepo.push(editHash, repo.url(), "master"); + TestBotRunner.runPeriodicItems(notifyBot); + + // The changeset should be reflected in a comment + var comments = issue.comments(); + assertEquals(1, comments.size()); + var comment = comments.get(0); + assertTrue(comment.body().contains(editHash.abbreviate())); + + // There should be no open issues + assertEquals(0, issueProject.issues().size()); + } + } } diff --git a/bots/pr/build.gradle b/bots/pr/build.gradle index 3f5f654ae..5b6bb071f 100644 --- a/bots/pr/build.gradle +++ b/bots/pr/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':bot') implementation project(':forge') implementation project(':issuetracker') diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CensusInstance.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CensusInstance.java index 0cdc77327..6ade0a6b6 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CensusInstance.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CensusInstance.java @@ -49,7 +49,7 @@ private CensusInstance(Census census, JCheckConfiguration configuration, Project private static Repository initialize(HostedRepository repo, String ref, Path folder) { try { - return Repository.materialize(folder, repo.url(), ref); + return Repository.materialize(folder, repo.url(), "+" + ref + ":pr_census_" + repo.name()); } catch (IOException e) { throw new RuntimeException("Failed to retrieve census to " + folder, e); } diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java index 9f691b5f7..f7e222220 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckRun.java @@ -23,9 +23,9 @@ package org.openjdk.skara.bots.pr; import org.openjdk.skara.forge.*; -import org.openjdk.skara.host.*; +import org.openjdk.skara.host.HostUser; import org.openjdk.skara.issuetracker.*; -import org.openjdk.skara.vcs.*; +import org.openjdk.skara.vcs.Commit; import org.openjdk.skara.vcs.openjdk.Issue; import java.io.*; @@ -137,7 +137,9 @@ private List botSpecificChecks() throws IOException { var sourceBranch = mergeSourceBranch(); if (sourceBranch.isPresent() && sourceRepo.isPresent()) { try { - var mergeSourceRepo = pr.repository().forge().repository(sourceRepo.get()); + var mergeSourceRepo = pr.repository().forge().repository(sourceRepo.get()).orElseThrow(() -> + new RuntimeException("Could not find repository " + sourceRepo.get()) + ); try { var sourceHash = prInstance.localRepo().fetch(mergeSourceRepo.url(), sourceBranch.get()); if (!prInstance.localRepo().isAncestor(commits.get(1).hash(), sourceHash)) { @@ -171,6 +173,7 @@ private void updateCheckBuilder(CheckBuilder checkBuilder, PullRequestCheckIssue if (visitor.isReadyForReview() && additionalErrors.isEmpty()) { checkBuilder.complete(true); } else { + checkBuilder.title("Required"); var summary = Stream.concat(visitor.getMessages().stream(), additionalErrors.stream()) .sorted() .map(m -> "- " + m) @@ -260,27 +263,36 @@ private Optional getReviewersList(List reviews) { } } - private String getStatusMessage(List reviews, PullRequestCheckIssueVisitor visitor) { + private String getStatusMessage(List comments, List reviews, PullRequestCheckIssueVisitor visitor) { var progressBody = new StringBuilder(); progressBody.append("## Progress\n"); progressBody.append(getChecksList(visitor)); var issue = Issue.fromString(pr.title()); if (issueProject != null && issue.isPresent()) { - progressBody.append("\n\n## Issue\n"); - var iss = issueProject.issue(issue.get().id()); - if (iss.isPresent()) { - progressBody.append("["); - progressBody.append(iss.get().id()); - progressBody.append("]("); - progressBody.append(iss.get().webUrl()); - progressBody.append("): "); - progressBody.append(iss.get().title()); - progressBody.append("\n"); - } else { - progressBody.append("⚠️ Failed to retrieve information on issue `"); - progressBody.append(issue.get().toString()); - progressBody.append("`.\n"); + var allIssues = new ArrayList(); + allIssues.add(issue.get()); + allIssues.addAll(SolvesTracker.currentSolved(pr.repository().forge().currentUser(), comments)); + progressBody.append("\n\n## Issue"); + if (allIssues.size() > 1) { + progressBody.append("s"); + } + progressBody.append("\n"); + for (var currentIssue : allIssues) { + var iss = issueProject.issue(currentIssue.id()); + if (iss.isPresent()) { + progressBody.append("["); + progressBody.append(iss.get().id()); + progressBody.append("]("); + progressBody.append(iss.get().webUrl()); + progressBody.append("): "); + progressBody.append(iss.get().title()); + progressBody.append("\n"); + } else { + progressBody.append("⚠️ Failed to retrieve information on issue `"); + progressBody.append(currentIssue.id()); + progressBody.append("`.\n"); + } } } @@ -448,33 +460,39 @@ private void updateMergeReadyComment(boolean isReady, String commitMessage, List private void checkStatus() { var checkBuilder = CheckBuilder.create("jcheck", pr.headHash()); - checkBuilder.title("Required"); var censusDomain = censusInstance.configuration().census().domain(); + Exception checkException = null; try { // Post check in-progress log.info("Starting to run jcheck on PR head"); pr.createCheck(checkBuilder.build()); var localHash = prInstance.commit(censusInstance.namespace(), censusDomain, null); - - // Try to rebase boolean rebasePossible = true; - var ignored = new PrintWriter(new StringWriter()); - var rebasedHash = prInstance.rebase(localHash, ignored); - if (rebasedHash.isEmpty()) { - rebasePossible = false; - } else { - localHash = rebasedHash.get(); - } + PullRequestCheckIssueVisitor visitor = prInstance.createVisitor(localHash, censusInstance); + List additionalErrors; + if (!localHash.equals(prInstance.baseHash())) { + // Try to rebase + var ignored = new PrintWriter(new StringWriter()); + var rebasedHash = prInstance.rebase(localHash, ignored); + if (rebasedHash.isEmpty()) { + rebasePossible = false; + } else { + localHash = rebasedHash.get(); + } - // Determine current status - var visitor = prInstance.executeChecks(localHash, censusInstance); - var additionalErrors = botSpecificChecks(); + // Determine current status + prInstance.executeChecks(localHash, censusInstance, visitor); + additionalErrors = botSpecificChecks(); + } + else { + additionalErrors = List.of("This PR contains no changes"); + } updateCheckBuilder(checkBuilder, visitor, additionalErrors); updateReadyForReview(visitor, additionalErrors); // Calculate and update the status message if needed - var statusMessage = getStatusMessage(activeReviews, visitor); + var statusMessage = getStatusMessage(comments, activeReviews, visitor); var updatedBody = updateStatusMessage(statusMessage); // Post / update approval messages (only needed if the review itself can't contain a body) @@ -513,11 +531,11 @@ private void checkStatus() { } catch (Exception e) { log.throwing("CommitChecker", "checkStatus", e); newLabels.remove("ready"); - var metadata = workItem.getMetadata(pr.title(), pr.body(), pr.comments(), activeReviews, newLabels, censusInstance, pr.targetHash()); - checkBuilder.metadata(metadata); - checkBuilder.title("Exception occurred during jcheck"); + checkBuilder.metadata("invalid"); + checkBuilder.title("Exception occurred during jcheck - the operation will be retried"); checkBuilder.summary(e.getMessage()); checkBuilder.complete(false); + checkException = e; } var check = checkBuilder.build(); pr.updateCheck(check); @@ -533,5 +551,10 @@ private void checkStatus() { pr.removeLabel(oldLabel); } } + + // After updating the PR, rethrow any exception to automatically retry on transient errors + if (checkException != null) { + throw new RuntimeException("Exception during jcheck", checkException); + } } } diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckWorkItem.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckWorkItem.java index 980965326..ed6ae8e4d 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckWorkItem.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/CheckWorkItem.java @@ -44,7 +44,7 @@ class CheckWorkItem extends PullRequestWorkItem { private final Map blockingLabels; private final IssueProject issueProject; - private final Pattern metadataComments = Pattern.compile(""; + private final static Pattern markerPattern = Pattern.compile(""); + + static String setSolvesMarker(Issue issue) { + var encodedDescription = Base64.getEncoder().encodeToString(issue.description().getBytes(StandardCharsets.UTF_8)); + return String.format(solvesMarker, issue.id(), encodedDescription); + } + + static String removeSolvesMarker(Issue issue) { + return String.format(solvesMarker, issue.id(), ""); + } + + static List currentSolved(HostUser botUser, List comments) { + var solvesActions = comments.stream() + .filter(comment -> comment.author().equals(botUser)) + .map(comment -> markerPattern.matcher(comment.body())) + .filter(Matcher::find) + .collect(Collectors.toList()); + var current = new LinkedHashMap(); + for (var action : solvesActions) { + var key = action.group(1); + if (action.group(2).equals("")) { + current.remove(key); + } else { + var decodedDescription = new String(Base64.getDecoder().decode(action.group(2)), StandardCharsets.UTF_8); + var issue = new Issue(key, decodedDescription); + current.put(key, issue); + } + } + + return new ArrayList<>(current.values()); + } +} diff --git a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SponsorCommand.java b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SponsorCommand.java index c09e1b4de..e9982548d 100644 --- a/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SponsorCommand.java +++ b/bots/pr/src/main/java/org/openjdk/skara/bots/pr/SponsorCommand.java @@ -87,7 +87,8 @@ public void handle(PullRequest pr, CensusInstance censusInstance, Path scratchPa } } - var issues = prInstance.executeChecks(localHash, censusInstance); + var issues = prInstance.createVisitor(localHash, censusInstance); + prInstance.executeChecks(localHash, censusInstance, issues); if (!issues.getMessages().isEmpty()) { reply.print("Your merge request cannot be fulfilled at this time, as "); reply.println("your changes failed the final jcheck:"); diff --git a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/CheckTests.java b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/CheckTests.java index 05cae8e43..8e441b786 100644 --- a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/CheckTests.java +++ b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/CheckTests.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.*; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.util.*; import java.util.logging.*; @@ -1027,4 +1028,126 @@ void draft(TestInfo testInfo) throws IOException { assertFalse(pr.labels().contains("ready")); } } + + @Test + void excessiveFailures(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var reviewer = credentials.getHostedRepository(); + + var censusBuilder = credentials.getCensusBuilder() + .addAuthor(author.forge().currentUser().id()) + .addReviewer(reviewer.forge().currentUser().id()); + var checkBot = new PullRequestBot(author, censusBuilder.build(), "master"); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR containing more errors than at least GitHub can handle in a check + var badContent = "\tline \n".repeat(200); + var editHash = CheckableRepository.appendAndCommit(localRepo, badContent); + localRepo.push(editHash, author.url(), "refs/heads/edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", + "This is a pull request", true); + + // Check the status + TestBotRunner.runPeriodicItems(checkBot); + + // Verify that the check failed + var checks = pr.checks(editHash); + assertEquals(1, checks.size()); + var check = checks.get("jcheck"); + assertEquals(CheckStatus.FAILURE, check.status()); + } + } + + @Test + void retryOnException(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var reviewer = credentials.getHostedRepository(); + + var censusBuilder = credentials.getCensusBuilder() + .addAuthor(author.forge().currentUser().id()) + .addReviewer(reviewer.forge().currentUser().id()); + var checkBot = new PullRequestBot(author, censusBuilder.build(), "master"); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Break the jcheck configuration + var confPath = tempFolder.path().resolve(".jcheck/conf"); + var oldConf = Files.readString(confPath, StandardCharsets.UTF_8); + Files.writeString(confPath, "Hello there!", StandardCharsets.UTF_8); + localRepo.add(confPath); + var editHash = CheckableRepository.appendAndCommit(localRepo, "A change"); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", + "This is a pull request", true); + + // Check the status - should throw every time + assertThrows(RuntimeException.class, () -> TestBotRunner.runPeriodicItems(checkBot)); + assertThrows(RuntimeException.class, () -> TestBotRunner.runPeriodicItems(checkBot)); + assertThrows(RuntimeException.class, () -> TestBotRunner.runPeriodicItems(checkBot)); + + // Verify that the check failed + var checks = pr.checks(editHash); + assertEquals(1, checks.size()); + var check = checks.get("jcheck"); + assertEquals(CheckStatus.FAILURE, check.status()); + + Files.writeString(confPath, oldConf, StandardCharsets.UTF_8); + localRepo.add(confPath); + editHash = CheckableRepository.appendAndCommit(localRepo, "Another change"); + localRepo.push(editHash, author.url(), "edit"); + + TestBotRunner.runPeriodicItems(checkBot); + + // Verify that the check now passes + checks = pr.checks(editHash); + assertEquals(1, checks.size()); + check = checks.get("jcheck"); + assertEquals(CheckStatus.SUCCESS, check.status()); + } + } + + @Test + void noCommit(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var reviewer = credentials.getHostedRepository(); + + var censusBuilder = credentials.getCensusBuilder() + .addAuthor(author.forge().currentUser().id()) + .addReviewer(reviewer.forge().currentUser().id()); + var checkBot = new PullRequestBot(author, censusBuilder.build(), "master"); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "master"); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Check the status + TestBotRunner.runPeriodicItems(checkBot); + + // Verify that the check failed + var checks = pr.checks(editHash); + assertEquals(1, checks.size()); + var check = checks.get("jcheck"); + assertEquals(CheckStatus.FAILURE, check.status()); + } + } } diff --git a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/IntegrateTests.java b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/IntegrateTests.java index d42d06931..fa873975c 100644 --- a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/IntegrateTests.java +++ b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/IntegrateTests.java @@ -30,6 +30,7 @@ import org.junit.jupiter.api.*; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.util.Set; import java.util.stream.Collectors; @@ -482,14 +483,16 @@ void autoRebase(TestInfo testInfo) throws IOException { @Test void retryOnFailure(TestInfo testInfo) throws IOException { try (var credentials = new HostCredentials(testInfo); - var tempFolder = new TemporaryDirectory()) { + var tempFolder = new TemporaryDirectory(); + var censusFolder = new TemporaryDirectory()) { var author = credentials.getHostedRepository(); var integrator = credentials.getHostedRepository(); var censusBuilder = credentials.getCensusBuilder() .addCommitter(author.forge().currentUser().id()) .addReviewer(integrator.forge().currentUser().id()); - var mergeBot = new PullRequestBot(integrator, censusBuilder.build(), "master"); + var censusRepo = censusBuilder.build(); + var mergeBot = new PullRequestBot(integrator, censusRepo, "master"); // Populate the projects repository var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); @@ -509,15 +512,20 @@ void retryOnFailure(TestInfo testInfo) throws IOException { // Let the bot check it TestBotRunner.runPeriodicItems(mergeBot); - // Pre-push to cause a failure - localRepo.push(editHash, author.url(), "master"); + // Break the census to cause an exception + var localCensus = Repository.materialize(censusFolder.path(), censusRepo.url(), "+master:current_census"); + var currentCensusHash = localCensus.resolve("current_census").orElseThrow(); + Files.writeString(censusFolder.path().resolve("contributors.xml"), "This is not xml", StandardCharsets.UTF_8); + localCensus.add(censusFolder.path().resolve("contributors.xml")); + var badCensusHash = localCensus.commit("Bad census update", "duke", "duke@openjdk.org"); + localCensus.push(badCensusHash, censusRepo.url(), "master", true); // Attempt a merge (without triggering another check) pr.addComment("/integrate"); assertThrows(RuntimeException.class, () -> TestBotRunner.runPeriodicItems(mergeBot, wi -> wi instanceof CheckWorkItem)); - // Restore the master branch - localRepo.push(masterHash, author.url(), "master", true); + // Restore the census + localCensus.push(currentCensusHash, censusRepo.url(), "master", true); // The bot should now retry TestBotRunner.runPeriodicItems(mergeBot, wi -> wi instanceof CheckWorkItem); diff --git a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SolvesTests.java b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SolvesTests.java new file mode 100644 index 000000000..65ca62add --- /dev/null +++ b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SolvesTests.java @@ -0,0 +1,279 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.pr; + +import org.junit.jupiter.api.*; +import org.openjdk.skara.forge.Review; +import org.openjdk.skara.issuetracker.Comment; +import org.openjdk.skara.test.*; +import org.openjdk.skara.vcs.Repository; + +import java.io.IOException; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; +import static org.openjdk.skara.bots.pr.PullRequestAsserts.assertLastCommentContains; + +class SolvesTests { + @Test + void simple(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var integrator = credentials.getHostedRepository(); + + var censusBuilder = credentials.getCensusBuilder() + .addReviewer(integrator.forge().currentUser().id()) + .addCommitter(author.forge().currentUser().id()); + var prBot = new PullRequestBot(integrator, censusBuilder.build(), "master"); + + // Populate the projects repository + var localRepoFolder = tempFolder.path().resolve("localrepo"); + var localRepo = CheckableRepository.init(localRepoFolder, author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + assertFalse(CheckableRepository.hasBeenEdited(localRepo)); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "123: This is a pull request"); + + // No arguments + pr.addComment("/solves"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a help message + assertLastCommentContains(pr,"To add an additional"); + + // Invalid syntax + pr.addComment("/solves something I guess"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a failure message + assertLastCommentContains(pr,"Invalid"); + + // Add an issue + pr.addComment("/solves 1234: An issue"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a success message + assertLastCommentContains(pr,"Adding additional"); + + // Try to remove a not-previously-added issue + pr.addComment("/solves 1235"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a failure message + assertLastCommentContains(pr,"Could not find"); + + // Now remove the added one + pr.addComment("/solves 1234"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a success message + assertLastCommentContains(pr,"Removing additional"); + + // Add two more issues + pr.addComment("/solves 12345: Another issue"); + pr.addComment("/solves 123456: Yet another issue"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a success message + assertLastCommentContains(pr,"Adding additional"); + + // Update the description of the first one + pr.addComment("/solves 12345: This is indeed another issue"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a success message + assertLastCommentContains(pr,"Updating description"); + + // Approve it as another user + var approvalPr = integrator.pullRequest(pr.id()); + approvalPr.addReview(Review.Verdict.APPROVED, "Approved"); + TestBotRunner.runPeriodicItems(prBot); + TestBotRunner.runPeriodicItems(prBot); + + // The commit message preview should contain the additional issues + var preview = pr.comments().stream() + .filter(comment -> comment.body().contains("The commit message will be")) + .map(Comment::body) + .findFirst() + .orElseThrow(); + assertTrue(preview.contains("123: This is a pull request")); + assertTrue(preview.contains("12345: This is indeed another issue")); + assertTrue(preview.contains("123456: Yet another issue")); + + // Integrate + pr.addComment("/integrate"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with an ok message + assertLastCommentContains(pr,"Pushed as commit"); + + // The change should now be present on the master branch + var pushedFolder = tempFolder.path().resolve("pushed"); + var pushedRepo = Repository.materialize(pushedFolder, author.url(), "master"); + assertTrue(CheckableRepository.hasBeenEdited(pushedRepo)); + + var headHash = pushedRepo.resolve("HEAD").orElseThrow(); + var headCommit = pushedRepo.commits(headHash.hex() + "^.." + headHash.hex()).asList().get(0); + + // The additional issues should be present in the commit message + assertEquals(List.of("123: This is a pull request", + "12345: This is indeed another issue", + "123456: Yet another issue", + "", + "Reviewed-by: integrationreviewer1"), headCommit.message()); + } + } + + @Test + void invalidCommandAuthor(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var integrator = credentials.getHostedRepository(); + var external = credentials.getHostedRepository(); + + var censusBuilder = credentials.getCensusBuilder() + .addAuthor(author.forge().currentUser().id()); + var mergeBot = new PullRequestBot(integrator, censusBuilder.build(), "master"); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + assertFalse(CheckableRepository.hasBeenEdited(localRepo)); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Issue a solves command not as the PR author + var externalPr = external.pullRequest(pr.id()); + externalPr.addComment("/solves 1234: an issue"); + TestBotRunner.runPeriodicItems(mergeBot); + + // The bot should reply with an error message + var error = pr.comments().stream() + .filter(comment -> comment.body().contains("Only the author")) + .count(); + assertEquals(1, error); + } + } + + @Test + void issueInTitle(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var integrator = credentials.getHostedRepository(); + + var censusBuilder = credentials.getCensusBuilder() + .addAuthor(author.forge().currentUser().id()); + var prBot = new PullRequestBot(integrator, censusBuilder.build(), "master"); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + assertFalse(CheckableRepository.hasBeenEdited(localRepo)); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request"); + + // Add an issue + pr.addComment("/solves 1234: An issue"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a success message + assertLastCommentContains(pr,"current title"); + + var updatedPr = author.pullRequest(pr.id()); + assertEquals("1234: An issue", updatedPr.title()); + + // Update the issue description + pr.addComment("/solves 1234: Yes this is an issue"); + TestBotRunner.runPeriodicItems(prBot); + + // The bot should reply with a success message + assertLastCommentContains(pr,"will now be updated"); + + updatedPr = author.pullRequest(pr.id()); + assertEquals("1234: Yes this is an issue", updatedPr.title()); + } + } + + @Test + void issueInBody(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tempFolder = new TemporaryDirectory()) { + var author = credentials.getHostedRepository(); + var integrator = credentials.getHostedRepository(); + var issues = credentials.getIssueProject(); + + var censusBuilder = credentials.getCensusBuilder() + .addAuthor(author.forge().currentUser().id()); + var prBot = new PullRequestBot(integrator, censusBuilder.build(), "master", + Map.of(), Map.of(), Map.of(), Set.of(), Map.of(), issues); + + // Populate the projects repository + var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType()); + var masterHash = localRepo.resolve("master").orElseThrow(); + assertFalse(CheckableRepository.hasBeenEdited(localRepo)); + localRepo.push(masterHash, author.url(), "master", true); + + // Make a change with a corresponding PR + var editHash = CheckableRepository.appendAndCommit(localRepo); + localRepo.push(editHash, author.url(), "edit", true); + var issue1 = issues.createIssue("First", List.of("Hello")); + var pr = credentials.createPullRequest(author, "master", "edit", + issue1.id() + ": This is a pull request"); + + // First check + TestBotRunner.runPeriodicItems(prBot); + assertTrue(pr.body().contains(issue1.id())); + assertTrue(pr.body().contains("First")); + assertTrue(pr.body().contains("## Issue\n")); + + // Add an extra issue + var issue2 = issues.createIssue("Second", List.of("There")); + pr.addComment("/solves " + issue2.id() + ": Description"); + + // Check that the body was updated + TestBotRunner.runPeriodicItems(prBot); + TestBotRunner.runPeriodicItems(prBot); + assertTrue(pr.body().contains(issue1.id())); + assertTrue(pr.body().contains("First")); + assertTrue(pr.body().contains(issue2.id())); + assertTrue(pr.body().contains("Second")); + assertFalse(pr.body().contains("## Issue\n")); + assertTrue(pr.body().contains("## Issues\n")); + } + } +} diff --git a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SummaryTests.java b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SummaryTests.java index 8b16e103c..467a698a2 100644 --- a/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SummaryTests.java +++ b/bots/pr/src/test/java/org/openjdk/skara/bots/pr/SummaryTests.java @@ -168,5 +168,4 @@ void invalidCommandAuthor(TestInfo testInfo) throws IOException { assertEquals(1, error); } } - } diff --git a/bots/submit/build.gradle b/bots/submit/build.gradle index 65c22f42f..03896f180 100644 --- a/bots/submit/build.gradle +++ b/bots/submit/build.gradle @@ -31,6 +31,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':bot') implementation project(':host') implementation project(':forge') diff --git a/bots/submit/src/main/java/org/openjdk/skara/bots/submit/SubmitBotWorkItem.java b/bots/submit/src/main/java/org/openjdk/skara/bots/submit/SubmitBotWorkItem.java index 8ffcaea34..ea8b5e825 100644 --- a/bots/submit/src/main/java/org/openjdk/skara/bots/submit/SubmitBotWorkItem.java +++ b/bots/submit/src/main/java/org/openjdk/skara/bots/submit/SubmitBotWorkItem.java @@ -84,7 +84,8 @@ public void run(Path scratchPath) { // Materialize the PR's target ref try { - var localRepo = Repository.materialize(prFolder, pr.repository().url(), pr.targetRef()); + var localRepo = Repository.materialize(prFolder, pr.repository().url(), + "+" + pr.targetRef() + ":submit_" + pr.repository().name()); var headHash = localRepo.fetch(pr.repository().url(), pr.headHash().hex()); var checkBuilder = CheckBuilder.create(executor.checkName(), headHash); diff --git a/bots/tester/build.gradle b/bots/tester/build.gradle new file mode 100644 index 000000000..ff27e6384 --- /dev/null +++ b/bots/tester/build.gradle @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +module { + name = 'org.openjdk.skara.bots.tester' + test { + requires 'org.junit.jupiter.api' + requires 'org.openjdk.skara.test' + requires 'org.openjdk.skara.host' + opens 'org.openjdk.skara.bots.tester' to 'org.junit.platform.commons' + } +} + +dependencies { + implementation project(':bot') + implementation project(':ci') + implementation project(':census') + implementation project(':forge') + implementation project(':host') + implementation project(':issuetracker') + implementation project(':json') + implementation project(':vcs') + + testImplementation project(':test') +} diff --git a/bots/tester/src/main/java/module-info.java b/bots/tester/src/main/java/module-info.java new file mode 100644 index 000000000..bbe26d952 --- /dev/null +++ b/bots/tester/src/main/java/module-info.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +module org.openjdk.skara.bots.tester { + requires org.openjdk.skara.bot; + requires org.openjdk.skara.vcs; + requires org.openjdk.skara.ci; + + requires java.logging; + + provides org.openjdk.skara.bot.BotFactory with org.openjdk.skara.bots.tester.TestBotFactory; +} diff --git a/bots/tester/src/main/java/org/openjdk/skara/bots/tester/Stage.java b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/Stage.java new file mode 100644 index 000000000..ba2ab8712 --- /dev/null +++ b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/Stage.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +enum Stage { + NA, + ERROR, + REQUESTED, + PENDING, + APPROVED, + STARTED, + CANCELLED, + FINISHED +} diff --git a/bots/tester/src/main/java/org/openjdk/skara/bots/tester/State.java b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/State.java new file mode 100644 index 000000000..e09a81689 --- /dev/null +++ b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/State.java @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.forge.PullRequest; +import org.openjdk.skara.issuetracker.Comment; + +class State { + private final Stage stage; + private final Comment requested; + private final Comment pending; + private final Comment approval; + private final Comment started; + private final Comment cancelled; + private final Comment finished; + + private State(Stage stage, Comment requested, + Comment pending, + Comment approval, + Comment started, + Comment cancelled, + Comment finished) { + this.stage = stage; + this.requested = requested; + this.pending = pending; + this.approval = approval; + this.started = started; + this.cancelled = cancelled; + this.finished = finished; + } + + Stage stage() { + return stage; + } + + Comment requested() { + return requested; + } + + Comment pending() { + return pending; + } + + Comment approval() { + return approval; + } + + Comment started() { + return started; + } + + Comment cancelled() { + return cancelled; + } + + Comment finished() { + return finished; + } + + static State from(PullRequest pr, String approverGroupId) { + Comment requested = null; + Comment pending = null; + Comment approval = null; + Comment started = null; + Comment cancelled = null; + Comment error = null; + Comment finished = null; + + var isApproved = false; + + var host = pr.repository().forge(); + var comments = pr.comments(); + var start = -1; + for (var i = comments.size() - 1; i >=0; i--) { + var comment = comments.get(i); + var lines = comment.body().split("\n"); + if (lines.length == 1 && + lines[0].startsWith("/test") && + !lines[0].startsWith("/test approve") && + !lines[0].startsWith("/test cancel")) { + requested = comment; + start = i; + break; + } + } + + if (requested != null) { + var applicable = comments.subList(start, comments.size()); + for (var comment : applicable) { + var body = comment.body(); + var author = comment.author(); + if (author.equals(host.currentUser())) { + var lines = body.split("\n"); + switch (lines[0]) { + case "": + pending = comment; + break; + case "": + started = comment; + break; + case "": + error = comment; + break; + case "": + finished = comment; + break; + } + } else if (body.equals("/test approve")) { + approval = comment; + if (host.isMemberOf(approverGroupId, author)) { + isApproved = true; + } + } else if (body.equals("/test cancel")) { + if (comment.author().equals(requested.author())) { + cancelled = comment; + } + } else if (body.startsWith("/test")) { + if (host.isMemberOf(approverGroupId, author)) { + isApproved = true; + } + } + } + } + + Stage stage = null; + if (error != null) { + stage = Stage.ERROR; + } else if (cancelled != null) { + stage = Stage.CANCELLED; + } else if (finished != null) { + stage = Stage.FINISHED; + } else if (started != null) { + stage = Stage.STARTED; + } else if (requested != null && isApproved) { + stage = Stage.APPROVED; + } else if (requested != null && pending != null) { + stage = Stage.PENDING; + } else if (requested != null) { + stage = Stage.REQUESTED; + } else { + stage = Stage.NA; + } + + return new State(stage, requested, pending, approval, started, cancelled, finished); + } +} diff --git a/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBot.java b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBot.java new file mode 100644 index 000000000..440b93a3d --- /dev/null +++ b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBot.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.ci.ContinuousIntegration; +import org.openjdk.skara.ci.Job; +import org.openjdk.skara.bot.*; +import org.openjdk.skara.forge.*; + +import java.io.*; +import java.nio.file.*; +import java.util.*; +import java.util.stream.Collectors; + +public class TestBot implements Bot { + private final ContinuousIntegration ci; + private final String approversGroupId; + private final List availableJobs; + private final List defaultJobs; + private final String name; + private final Path storage; + private final HostedRepository repo; + private final PullRequestUpdateCache cache; + private final Set seen; + + TestBot(ContinuousIntegration ci, + String approversGroupId, + List availableJobs, + List defaultJobs, + String name, + Path storage, + HostedRepository repo) { + this.ci = ci; + this.approversGroupId = approversGroupId; + this.availableJobs = availableJobs; + this.defaultJobs = defaultJobs; + this.name = name; + this.storage = storage; + this.repo = repo; + this.cache = new PullRequestUpdateCache(); + this.seen = new HashSet<>(); + } + + @Override + public List getPeriodicItems() { + var ret = new ArrayList(); + + var host = repo.webUrl().getHost(); + var repoId = Long.toString(repo.id()); + for (var pr : repo.pullRequests()) { + if (cache.needsUpdate(pr)) { + ret.add(new TestWorkItem(ci, + approversGroupId, + availableJobs, + defaultJobs, + name, + storage, + pr)); + } else { + // is there a job running for this PR? + var colon = "%3A"; + var asterisk = "%2A"; + var id = host + "-" + repoId + "-"+ pr.id() + "-" + asterisk; + try { + var jobs = ci.query("id" + colon + id); + if (!jobs.isEmpty()) { + if (jobs.stream().anyMatch(j -> j.isRunning() || !seen.contains(j.id()))) { + ret.add(new TestWorkItem(ci, + approversGroupId, + availableJobs, + defaultJobs, + name, + storage, + pr)); + } + seen.addAll(jobs.stream().map(Job::id).collect(Collectors.toList())); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + + return ret; + } +} diff --git a/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBotFactory.java b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBotFactory.java new file mode 100644 index 000000000..9ce76b6eb --- /dev/null +++ b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestBotFactory.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.bot.*; +import org.openjdk.skara.json.*; + +import org.openjdk.skara.ci.ContinuousIntegration; + +import java.io.*; +import java.nio.file.*; +import java.util.*; +import java.util.stream.Collectors; +import java.net.URI; + +public class TestBotFactory implements BotFactory { + @Override + public String name() { + return "test"; + } + + @Override + public List create(BotConfiguration configuration) { + var storage = configuration.storageFolder(); + try { + Files.createDirectories(storage); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + var ret = new ArrayList(); + var specific = configuration.specific(); + + var approvers = specific.get("approvers").asString(); + var availableJobs = specific.get("availableJobs").stream().map(JSONValue::asString).collect(Collectors.toList()); + var defaultJobs = specific.get("defaultJobs").stream().map(JSONValue::asString).collect(Collectors.toList()); + var name = specific.get("name").asString(); + var ci = configuration.continuousIntegration(specific.get("ci").asString()); + for (var repo : specific.get("repositories").asArray()) { + var hostedRepo = configuration.repository(repo.asString()); + ret.add(new TestBot(ci, approvers, availableJobs, defaultJobs, name, storage, hostedRepo)); + } + + return ret; + } +} diff --git a/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestWorkItem.java b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestWorkItem.java new file mode 100644 index 000000000..af29dc586 --- /dev/null +++ b/bots/tester/src/main/java/org/openjdk/skara/bots/tester/TestWorkItem.java @@ -0,0 +1,443 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.bot.*; +import org.openjdk.skara.ci.*; +import org.openjdk.skara.forge.*; +import org.openjdk.skara.vcs.*; + +import java.io.*; +import java.net.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.time.ZonedDateTime; +import java.util.*; +import java.util.logging.Logger; +import java.util.stream.*; + +public class TestWorkItem implements WorkItem { + private final Logger log = Logger.getLogger("org.openjdk.skara.bots");; + private final ContinuousIntegration ci; + private final String approversGroupId; + private final List availableJobs; + private final List defaultJobs; + private final String name; + private final Path storage; + private final HostedRepository repository; + private final PullRequest pr; + + TestWorkItem(ContinuousIntegration ci, String approversGroupId, List availableJobs, + List defaultJobs, String name, Path storage, PullRequest pr) { + this.ci = ci; + this.approversGroupId = approversGroupId; + this.availableJobs = availableJobs; + this.defaultJobs = defaultJobs; + this.name = name; + this.storage = storage; + this.repository = pr.repository(); + this.pr = pr; + } + + @Override + public boolean concurrentWith(WorkItem other) { + if (!(other instanceof TestWorkItem)) { + return true; + } + var o = (TestWorkItem) other; + if (!repository.url().equals(o.repository.url())) { + return true; + } + return !pr.id().equals(o.pr.id()); + } + + + private String jobId(State state) { + var host = repository.webUrl().getHost(); + return host + "-" + + Long.toString(repository.id()) + "-"+ + pr.id() + "-" + + state.requested().id(); + } + + + private String osDisplayName(Build.OperatingSystem os) { + switch (os) { + case WINDOWS: + return "Windows"; + case MACOS: + return "macOS"; + case LINUX: + return "Linux"; + case SOLARIS: + return "Solaris"; + case AIX: + return "AIX"; + case FREEBSD: + return "FreeBSD"; + case OPENBSD: + return "OpenBSD"; + case NETBSD: + return "NetBSD"; + case HPUX: + return "HP-UX"; + case HAIKU: + return "Haiku"; + default: + throw new IllegalArgumentException("Unknown operating system: " + os.toString()); + } + } + + private String cpuDisplayName(Build.CPU cpu) { + switch (cpu) { + case X86: + return "x86"; + case X64: + return "x64"; + case SPARCV9: + return "SPARC V9"; + case AARCH64: + return "AArch64"; + case AARCH32: + return "AArch32"; + case PPCLE32: + return "PPC LE 32"; + case PPCLE64: + return "PPC LE 64"; + default: + throw new IllegalArgumentException("Unknown cpu: " + cpu.toString()); + } + } + + private String debugLevelDisplayName(Build.DebugLevel level) { + switch (level) { + case RELEASE: + return "release"; + case FASTDEBUG: + return "fastdebug"; + case SLOWDEBUG: + return "slowdebug"; + default: + throw new IllegalArgumentException("Unknown debug level: " + level.toString()); + } + } + + private void appendIdSection(StringBuilder summary, Job job) { + summary.append("## Id"); + summary.append("\n"); + + summary.append("`"); + summary.append(job.id()); + summary.append("`"); + summary.append("\n"); + } + + private void appendBuildsSection(StringBuilder summary, Job job) { + var perOSandArch = new HashMap>(); + for (var build : job.builds()) { + var osAndArch = osDisplayName(build.os()) + " " + cpuDisplayName(build.cpu()); + var debugLevel = debugLevelDisplayName(build.debugLevel()); + if (!perOSandArch.containsKey(osAndArch)) { + perOSandArch.put(osAndArch, new ArrayList()); + } + perOSandArch.get(osAndArch).add(debugLevel); + } + + summary.append("## Builds"); + summary.append("\n"); + + for (var key : perOSandArch.keySet()) { + summary.append("- "); + summary.append(key); + summary.append(" ("); + summary.append(String.join(",", perOSandArch.get(key))); + summary.append(")"); + summary.append("\n"); + } + } + + private void appendTestsSection(StringBuilder summary, Job job) { + summary.append("## Tests"); + summary.append("\n"); + + for (var test : job.tests()) { + summary.append("- "); + summary.append(test.name()); + summary.append("\n"); + } + } + + private void appendStatusSection(StringBuilder summary, Job job) { + var s = job.status(); + summary.append("## Status"); + summary.append("\n"); + + var numCompleted = s.numCompleted(); + summary.append(Integer.toString(numCompleted)); + summary.append(numCompleted == 1 ? " job " : " jobs "); + summary.append("completed, "); + + var numRunning = s.numRunning(); + summary.append(Integer.toString(numRunning)); + summary.append(numRunning == 1 ? " job " : " jobs "); + summary.append("running, "); + + var numNotStarted = s.numNotStarted(); + summary.append(Integer.toString(numNotStarted)); + summary.append(numNotStarted == 1 ? " job " : " jobs "); + summary.append("not yet started"); + summary.append("\n"); + } + + private void appendResultSection(StringBuilder summary, Job job) { + var r = job.result(); + summary.append("## Result"); + summary.append("\n"); + + var numPassed = r.numPassed(); + summary.append(Integer.toString(numPassed)); + summary.append(numPassed == 1 ? " job " : " jobs "); + summary.append("passed, "); + + var numFailed = r.numFailed(); + summary.append(Integer.toString(numFailed)); + summary.append(numFailed == 1 ? " job " : " jobs "); + summary.append("with failures, "); + + var numSkipped = r.numSkipped(); + summary.append(Integer.toString(numSkipped)); + summary.append(numSkipped == 1 ? " job " : " jobs "); + summary.append("not run"); + summary.append("\n"); + } + + private String display(Job job) { + var sb = new StringBuilder(); + appendIdSection(sb, job); + sb.append("\n"); + appendBuildsSection(sb, job); + sb.append("\n"); + appendTestsSection(sb, job); + sb.append("\n"); + appendStatusSection(sb, job); + sb.append("\n"); + if (job.isCompleted()) { + appendResultSection(sb, job); + } + return sb.toString(); + } + + @Override + public void run(Path scratchPath) { + var state = State.from(pr, approversGroupId); + var stage = state.stage(); + if (stage == Stage.NA || stage == Stage.ERROR || stage == Stage.PENDING || stage == Stage.FINISHED) { + // nothing to do + return; + } + + if (stage == Stage.STARTED) { + if (state.started() != null) { + var lines = state.started().body().split("\n"); + var jobId = lines[1].replace("", ""); + var hash = lines[2].replace("", ""); + + try { + var job = ci.job(jobId); + var checks = pr.checks(new Hash(hash)); + if (checks.containsKey(name)) { + var check = checks.get(name); + if (check.status() == CheckStatus.IN_PROGRESS) { + var builder = CheckBuilder.from(check); + if (job.isCompleted()) { + var success = job.result().numFailed() == 0 && + job.result().numSkipped() == 0; + builder = builder.complete(success); + var requestor = state.requested().author().userName(); + var commentLines = List.of( + "", + "", + "", + "@" + requestor + " your test job with id " + jobId + " for commits up until " + hash.substring(0, 8) + " has finished." + ); + pr.addComment(String.join("\n", commentLines)); + } + builder = builder.summary(display(job)); + pr.updateCheck(builder.build()); + } + } else { + log.warning("Could not find check for job with " + jobId + " for hash " + hash + " for PR " + pr.webUrl()); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + log.warning("No 'started' comment present for PR " + pr.webUrl()); + } + } else if (stage == stage.CANCELLED) { + if (state.started() != null) { + var lines = state.started().body().split("\n"); + var jobId = lines[1].replace("", ""); + var hash = lines[2].replace("", ""); + + try { + ci.cancel(jobId); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + var checks = pr.checks(new Hash(hash)); + if (checks.containsKey(name)) { + var check = checks.get(name); + if (check.status() != CheckStatus.CANCELLED) { + var builder = CheckBuilder.from(check); + var newCheck = builder.cancel() + .build(); + pr.updateCheck(newCheck); + } + } else { + log.warning("Could not find check for job with " + jobId + " for hash " + hash + " for PR " + pr.webUrl()); + } + } + } else if (stage == Stage.REQUESTED) { + var requestedJobs = state.requested().body().substring("/test".length()); + if (requestedJobs.trim().isEmpty()) { + requestedJobs = String.join(",", defaultJobs); + } + var trimmedJobs = Stream.of(requestedJobs.split(",")).map(String::trim).collect(Collectors.toList()); + var nonExistingJobs = trimmedJobs.stream().filter(s -> !availableJobs.contains(s)) + .collect(Collectors.toList()); + if (!nonExistingJobs.isEmpty()) { + var wording = nonExistingJobs.size() == 1 ? "group " : "groups "; + var lines = List.of( + "", + "@" + state.requested().author().userName() + " the test " + wording + String.join(",", nonExistingJobs) + " does not exist" + ); + pr.addComment(String.join("\n", lines)); + } else { + var head = pr.headHash(); + var lines = List.of( + "", + "", + "", + "@" + state.requested().author().userName() + " you need to get approval to run the tests in " + + String.join(",", trimmedJobs) + " for commits up until " + head.abbreviate() + ); + pr.addComment(String.join("\n", lines)); + } + } else if (stage == Stage.APPROVED) { + Hash head = null; + List jobs = null; + + if (state.pending() != null) { + var comment = state.pending(); + var body = comment.body().split("\n"); + + head = new Hash(body[1].replace("", "")); + var requestedJobs = body[2].replace("", ""); + jobs = Arrays.asList(requestedJobs.split(",")); + } else { + var comment = state.requested(); + var body = comment.body().split("\n"); + + head = pr.headHash(); + var requestedJobs = state.requested().body().substring("/test".length()); + if (requestedJobs.trim().isEmpty()) { + requestedJobs = String.join(",", defaultJobs); + } + var trimmedJobs = Stream.of(requestedJobs.split(",")).map(String::trim).collect(Collectors.toList()); + var nonExistingJobs = trimmedJobs.stream().filter(s -> !availableJobs.contains(s)) + .collect(Collectors.toList()); + if (!nonExistingJobs.isEmpty()) { + var wording = nonExistingJobs.size() == 1 ? "group " : "groups "; + var lines = List.of( + "", + "@" + state.requested().author().userName() + " the test " + wording + String.join(",", nonExistingJobs) + " does not exist" + ); + pr.addComment(String.join("\n", lines)); + return; + } + + jobs = trimmedJobs; + } + var jobId = jobId(state); + + Job job = null; + Hash fetchHead = null; + try { + var sanitizedUrl = URLEncoder.encode(repository.webUrl().toString(), StandardCharsets.UTF_8); + var localRepoDir = storage.resolve("mach5-bot") + .resolve(sanitizedUrl) + .resolve(pr.id()); + var host = repository.webUrl().getHost(); + Repository localRepo = null; + if (!Files.exists(localRepoDir)) { + log.info("Cloning " + repository.name()); + Files.createDirectories(localRepoDir); + var url = repository.webUrl().toString(); + if (!url.endsWith(".git")) { + url += ".git"; + } + localRepo = Repository.clone(URI.create(url), localRepoDir); + } else { + log.info("Found existing scratch directory for " + repository.name()); + localRepo = Repository.get(localRepoDir).orElseThrow(() -> { + return new RuntimeException("Repository in " + localRepoDir + " has vanished"); + }); + } + fetchHead = localRepo.fetch(repository.url(), pr.targetRef()); + localRepo.checkout(fetchHead, true); + job = ci.submit(localRepoDir, jobs, jobId); + } catch (IOException e) { + var lines = List.of( + "", + "Could not create test job" + ); + pr.addComment(String.join("\n", lines)); + + throw new UncheckedIOException(e); + } + + var check = CheckBuilder.create(name, fetchHead) + .title("Summary") + .summary(display(job)) + .metadata(jobId) + .build(); + pr.createCheck(check); + + var lines = List.of( + "", + "", + "", + "A test job has been started with id: " + jobId + ); + pr.addComment(String.join("\n", lines)); + } else { + throw new RuntimeException("Unexpected state " + state); + } + } + + @Override + public String toString() { + return "TestWorkItem@" + pr.repository().name() + "#" + pr.id(); + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryContinuousIntegration.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryContinuousIntegration.java new file mode 100644 index 000000000..280b049db --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryContinuousIntegration.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.ci.ContinuousIntegration; +import org.openjdk.skara.ci.Job; +import org.openjdk.skara.host.HostUser; + +import java.io.*; +import java.nio.file.*; +import java.util.*; + +class InMemoryContinuousIntegration implements ContinuousIntegration { + static class Submission { + Path source; + List jobs; + String id; + + Submission(Path source, List jobs, String id) { + this.source = source; + this.jobs = jobs; + this.id = id; + } + } + + List submissions = new ArrayList(); + List cancelled = new ArrayList(); + Map jobs = new HashMap<>(); + boolean throwOnSubmit = false; + boolean isValid = true; + Map users = new HashMap<>(); + HostUser currentUser = null; + Map> groups = new HashMap<>(); + + @Override + public boolean isValid() { + return isValid; + } + + @Override + public HostUser user(String username) { + return users.get(username); + } + + @Override + public HostUser currentUser() { + return currentUser; + } + + @Override + public boolean isMemberOf(String groupId, HostUser user) { + var group = groups.get(groupId); + return group != null && group.contains(user); + } + + @Override + public Job submit(Path source, List jobs, String id) throws IOException { + if (throwOnSubmit) { + throw new IOException("Something went wrong"); + } + submissions.add(new Submission(source, jobs, id)); + return job(id); + } + + @Override + public Job job(String id) throws IOException { + return jobs.get(id); + } + + @Override + public void cancel(String id) throws IOException { + cancelled.add(id); + } + + @Override + public List query(String query) throws IOException { + return List.of(); + } +} diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraHost.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHost.java similarity index 56% rename from issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraHost.java rename to bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHost.java index 682a41df5..acda49f91 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraHost.java +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHost.java @@ -20,56 +20,44 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.issuetracker; +package org.openjdk.skara.bots.tester; +import org.openjdk.skara.forge.*; import org.openjdk.skara.host.*; -import org.openjdk.skara.network.*; -import org.openjdk.skara.json.JSON; -import java.net.URI; +import java.util.*; -public class JiraHost implements IssueTracker { - private final URI uri; - private final RestRequest request; - - public JiraHost(URI uri) { - this.uri = uri; - - var baseApi = URIBuilder.base(uri) - .setPath("/rest/api/2/") - .build(); - request = new RestRequest(baseApi); - } - - URI getUri() { - return uri; - } +class InMemoryHost implements Forge { + HostUser currentUserDetails = new HostUser(0, "openjdk", "openjdk [bot]"); + Map> groups; @Override public boolean isValid() { - var version = request.get("serverInfo") - .onError(r -> JSON.object().put("invalid", true)) - .execute(); - return !version.contains("invalid"); + return false; } @Override - public IssueProject project(String name) { - return new JiraProject(this, request, name); + public Optional repository(String name) { + return Optional.empty(); } @Override public HostUser user(String username) { - throw new RuntimeException("needs authentication; not implemented yet"); + return null; } @Override public HostUser currentUser() { - throw new RuntimeException("needs authentication; not implemented yet"); + return currentUserDetails; + } + + @Override + public boolean supportsReviewBody() { + return false; } @Override public boolean isMemberOf(String groupId, HostUser user) { - throw new RuntimeException("not implemented yet"); + return groups.get(groupId).contains(user); } } diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHostedRepository.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHostedRepository.java new file mode 100644 index 000000000..98e6f9dec --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryHostedRepository.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.forge.*; +import org.openjdk.skara.json.JSONValue; +import org.openjdk.skara.vcs.*; + +import java.net.URI; +import java.util.*; + +class InMemoryHostedRepository implements HostedRepository { + Forge host; + URI webUrl; + URI url; + long id; + + @Override + public Forge forge() { + return host; + } + + @Override + public PullRequest createPullRequest(HostedRepository target, + String targetRef, + String sourceRef, + String title, + List body, + boolean draft) { + return null; + } + + @Override + public PullRequest pullRequest(String id) { + return null; + } + + @Override + public List pullRequests() { + return null; + } + + @Override + public Optional parsePullRequestUrl(String url) { + return null; + } + + @Override + public String name() { + return null; + } + + @Override + public Optional parent() { + return null; + } + + @Override + public URI url() { + return url; + } + + @Override + public URI webUrl() { + return webUrl; + } + + @Override + public URI webUrl(Hash hash) { + return null; + } + + @Override + public VCS repositoryType() { + return null; + } + + @Override + public String fileContents(String filename, String ref) { + return null; + } + + @Override + public String namespace() { + return null; + } + + @Override + public Optional parseWebHook(JSONValue body) { + return null; + } + + @Override + public HostedRepository fork() { + return null; + } + + @Override + public long id() { + return id; + } + + @Override + public Hash branchHash(String ref) { + return null; + } + + @Override + public List findPullRequestsWithComment(String author, String body) { + return null; + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryJob.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryJob.java new file mode 100644 index 000000000..c9ecdd262 --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryJob.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.ci.*; + +import java.util.List; +import java.util.ArrayList; + +class InMemoryJob implements Job { + String id = ""; + List builds = new ArrayList<>(); + List tests = new ArrayList<>(); + Job.Status status; + Job.Result result; + Job.State state; + + @Override + public String id() { + return id; + } + + @Override + public List builds() { + return builds; + } + + @Override + public List tests() { + return tests; + } + + @Override + public Job.Status status() { + return status; + } + + @Override + public Job.Result result() { + return result; + } + + @Override + public Job.State state() { + return state; + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryPullRequest.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryPullRequest.java new file mode 100644 index 000000000..a19b41034 --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/InMemoryPullRequest.java @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.forge.*; +import org.openjdk.skara.host.*; +import org.openjdk.skara.issuetracker.*; +import org.openjdk.skara.vcs.*; + +import java.util.*; +import java.time.*; +import java.net.*; + +class InMemoryPullRequest implements PullRequest { + List comments = new ArrayList(); + List reviews = new ArrayList(); + HostUser author; + HostedRepository repository; + Hash headHash; + String id; + String targetRef; + Map> checks = new HashMap<>(); + + @Override + public HostedRepository repository() { + return repository; + } + + @Override + public String id() { + return id; + } + + @Override + public HostUser author() { + return author; + } + + @Override + public List reviews() { + return reviews; + } + + @Override + public void addReview(Review.Verdict verdict, String body) { + } + + @Override + public ReviewComment addReviewComment(Hash base, Hash hash, String path, int line, String body) { + return null; + } + + @Override + public ReviewComment addReviewCommentReply(ReviewComment parent, String body) { + return null; + } + + @Override + public List reviewComments() { + return null; + } + + @Override + public Hash headHash() { + return headHash; + } + + @Override + public String sourceRef() { + return null; + } + + @Override + public String targetRef() { + return targetRef; + } + + @Override + public Hash targetHash() { + return null; + } + + @Override + public String title() { + return null; + } + + @Override + public String body() { + return null; + } + + @Override + public void setBody(String body) { + } + + @Override + public List comments() { + return comments; + } + void setComments(List comments) { + this.comments = comments; + } + + @Override + public Comment addComment(String body) { + var user = repository().forge().currentUser(); + var now = ZonedDateTime.now(); + var id = comments.size(); + var comment = new Comment(Integer.toString(id), body, user, now, now); + comments.add(comment); + return comment; + } + + @Override + public Comment updateComment(String id, String body) { + var index = Integer.parseInt(id); + var old = comments.get(index); + + var now = ZonedDateTime.now(); + var newComment = new Comment(id, body, old.author(), old.createdAt(), now); + comments.set(index, newComment); + return newComment; + } + + @Override + public ZonedDateTime createdAt() { + return null; + } + + @Override + public ZonedDateTime updatedAt() { + return null; + } + + @Override + public Map checks(Hash hash) { + return checks.get(hash.hex()); + } + + @Override + public void createCheck(Check check) { + if (!checks.containsKey(check.hash().hex())) { + checks.put(check.hash().hex(), new HashMap<>()); + } + checks.get(check.hash().hex()).put(check.name(), check); + } + + @Override + public void updateCheck(Check check) { + if (checks.containsKey(check.hash().hex())) { + checks.get(check.hash().hex()).put(check.name(), check); + } + } + + @Override + public URI changeUrl() { + return null; + } + + @Override + public URI changeUrl(Hash base) { + return null; + } + + @Override + public boolean isDraft() { + return false; + } + + @Override + public void setState(State state) { + } + + @Override + public void addLabel(String label) { + } + + @Override + public void removeLabel(String label) { + } + + @Override + public List labels() { + return null; + } + + @Override + public URI webUrl() { + return null; + } + + @Override + public List assignees() { + return null; + } + + @Override + public void setAssignees(List assignees) { + } + + @Override + public void setTitle(String title) { + } + + @Override + public IssueProject project() { + return null; + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/StateTests.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/StateTests.java new file mode 100644 index 000000000..2f1d54eb2 --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/StateTests.java @@ -0,0 +1,391 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.issuetracker.Comment; +import org.openjdk.skara.host.HostUser; + +import java.util.*; +import java.time.ZonedDateTime; + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +class StateTests { + @Test + void noCommentsShouldEqualNA() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.comments = List.of(); + + var state = State.from(pr, "0"); + assertEquals(Stage.NA, state.stage()); + assertEquals(null, state.requested()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void testCommentFromNotApprovedUserShouldEqualRequested() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = List.of(comment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of()); + + var state = State.from(pr, approvers); + assertEquals(Stage.REQUESTED, state.stage()); + assertEquals(comment, state.requested()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void testCommentFromApprovedUserShouldEqualApproved() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = List.of(comment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of(duke)); + + var state = State.from(pr, approvers); + assertEquals(Stage.APPROVED, state.stage()); + assertEquals(comment, state.requested()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void testApprovalNeededCommentShouldResultInPending() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + + var pendingBody = List.of( + "", + "", + "@duke you need to get approval to run these tests" + ); + var pendingComment = new Comment("0", String.join("\n", pendingBody), bot, now, now); + pr.comments = List.of(testComment, pendingComment); + host.groups = Map.of("0", Set.of()); + + var state = State.from(pr, "0"); + assertEquals(Stage.PENDING, state.stage()); + assertEquals(testComment, state.requested()); + assertEquals(pendingComment, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void testStartedCommentShouldResultInRunning() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + + var pendingBody = List.of( + "", + "", + "@duke you need to get approval to run these tests" + ); + var pendingComment = new Comment("1", String.join("\n", pendingBody), bot, now, now); + + var member = new HostUser(2, "foo", "Foo Bar"); + var approveComment = new Comment("2", "/test approve", member, now, now); + + var startedBody = List.of( + "", + "", + "A test job has been started with id 0" + ); + var startedComment = new Comment("3", String.join("\n", startedBody), bot, now, now); + + pr.comments = List.of(testComment, pendingComment, approveComment, startedComment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of(member)); + + var state = State.from(pr, approvers); + assertEquals(Stage.STARTED, state.stage()); + assertEquals(testComment, state.requested()); + assertEquals(pendingComment, state.pending()); + assertEquals(startedComment, state.started()); + } + + @Test + void cancelCommentFromAuthorShouldEqualCancelled() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + var cancelComment = new Comment("1", "/test cancel", duke, now, now); + pr.comments = List.of(testComment, cancelComment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of()); + + var state = State.from(pr, approvers); + assertEquals(Stage.CANCELLED, state.stage()); + assertEquals(testComment, state.requested()); + assertEquals(cancelComment, state.cancelled()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void cancelCommentFromAnotherUserShouldHaveNoEffect() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var user = new HostUser(0, "foo", "Foo Bar"); + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + var cancelComment = new Comment("1", "/test cancel", user, now, now); + pr.comments = List.of(testComment, cancelComment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of()); + + var state = State.from(pr, approvers); + assertEquals(Stage.REQUESTED, state.stage()); + assertEquals(testComment, state.requested()); + assertEquals(null, state.cancelled()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void multipleTestCommentsShouldOnlyCareAboutLast() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var test1Comment = new Comment("0", "/test tier1", duke, now, now); + var test2Comment = new Comment("1", "/test tier1,tier2", duke, now, now); + var test3Comment = new Comment("2", "/test tier1,tier2,tier3", duke, now, now); + pr.comments = List.of(test1Comment, test2Comment, test3Comment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of()); + + var state = State.from(pr, approvers); + assertEquals(Stage.REQUESTED, state.stage()); + assertEquals(test3Comment, state.requested()); + assertEquals(null, state.cancelled()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void errorAfterRequestedShouldBeError() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + + var lines = List.of( + "", + "The test tier1 does not exist" + ); + var errorComment = new Comment("2", String.join("\n", lines), bot, now, now); + pr.comments = List.of(testComment, errorComment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of()); + + var state = State.from(pr, approvers); + assertEquals(Stage.ERROR, state.stage()); + assertEquals(testComment, state.requested()); + assertEquals(null, state.pending()); + assertEquals(null, state.started()); + } + + @Test + void testFinishedCommentShouldResultInFinished() { + var bot = new HostUser(1, "bot", "openjdk [bot]"); + + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + + var pendingBody = List.of( + "", + "", + "@duke you need to get approval to run these tests" + ); + var pendingComment = new Comment("1", String.join("\n", pendingBody), bot, now, now); + + var member = new HostUser(2, "foo", "Foo Bar"); + var approveComment = new Comment("2", "/test approve", member, now, now); + + var startedBody = List.of( + "", + "", + "A test job has been started with id 0" + ); + var startedComment = new Comment("3", String.join("\n", startedBody), bot, now, now); + + var finishedBody = List.of( + "", + "", + "A test job has been started with id 0" + ); + var finishedComment = new Comment("4", String.join("\n", finishedBody), bot, now, now); + + pr.comments = List.of(testComment, pendingComment, approveComment, startedComment, finishedComment); + + var approvers = "0"; + host.groups = Map.of(approvers, Set.of(member)); + + var state = State.from(pr, approvers); + assertEquals(Stage.FINISHED, state.stage()); + assertEquals(testComment, state.requested()); + assertEquals(pendingComment, state.pending()); + assertEquals(startedComment, state.started()); + assertEquals(finishedComment, state.finished()); + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestBotTests.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestBotTests.java new file mode 100644 index 000000000..d8ca2c4a7 --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestBotTests.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.test.*; + +import java.io.*; +import java.util.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import static org.junit.jupiter.api.Assertions.*; + +class TestBotTests { + @Test + void noTestCommentShouldDoNothing(TestInfo testInfo) throws IOException { + try (var credentials = new HostCredentials(testInfo); + var tmp = new TemporaryDirectory()) { + var upstreamHostedRepo = credentials.getHostedRepository(); + var personalHostedRepo = credentials.getHostedRepository(); + var pr = personalHostedRepo.createPullRequest(upstreamHostedRepo, + "master", + "master", + "Title", + List.of("body")); + + var comments = pr.comments(); + assertEquals(0, comments.size()); + + var storage = tmp.path().resolve("storage"); + var ci = new InMemoryContinuousIntegration(); + var bot = new TestBot(ci, "0", List.of(), List.of(), "", storage, upstreamHostedRepo); + var runner = new TestBotRunner(); + + runner.runPeriodicItems(bot); + + comments = pr.comments(); + assertEquals(0, comments.size()); + } + } +} diff --git a/bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestWorkItemTests.java b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestWorkItemTests.java new file mode 100644 index 000000000..4f1c0133b --- /dev/null +++ b/bots/tester/src/test/java/org/openjdk/skara/bots/tester/TestWorkItemTests.java @@ -0,0 +1,952 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.tester; + +import org.openjdk.skara.forge.CheckStatus; +import org.openjdk.skara.host.*; +import org.openjdk.skara.issuetracker.Comment; +import org.openjdk.skara.vcs.*; +import org.openjdk.skara.test.*; +import org.openjdk.skara.ci.Job; + +import java.io.*; +import java.net.URI; +import java.nio.file.*; +import java.util.*; +import java.time.ZonedDateTime; + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +class TestWorkItemTests { + @Test + void noTestCommentsShouldDoNothing() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.comments = List.of(); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + item.run(scratch); + + var comments = pr.comments(); + assertEquals(0, comments.size()); + } + } + + @Test + void topLevelTestApproveShouldDoNothing() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + var now = ZonedDateTime.now(); + pr.author = duke; + var testApproveComment = new Comment("0", "/test approve", duke, now, now); + pr.comments = List.of(testApproveComment); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + item.run(scratch); + + var comments = pr.comments(); + assertEquals(1, comments.size()); + assertEquals(testApproveComment, comments.get(0)); + } + } + + @Test + void topLevelTestCancelShouldDoNothing() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + var now = ZonedDateTime.now(); + pr.author = duke; + var testApproveComment = new Comment("0", "/test cancel", duke, now, now); + pr.comments = List.of(testApproveComment); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + item.run(scratch); + + var comments = pr.comments(); + assertEquals(1, comments.size()); + assertEquals(testApproveComment, comments.get(0)); + } + } + + @Test + void testCommentWithMadeUpJobShouldBeError() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of("0", Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test foobar", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals(2, lines.length); + assertEquals("", lines[0]); + assertEquals("@duke the test group foobar does not exist", lines[1]); + } + } + + @Test + void testCommentFromUnapprovedUserShouldBePending() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of("0", Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = new Hash("01234567890123456789012345789012345789"); + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test foobar", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + // Non-existing test group should result in error + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals(2, lines.length); + assertEquals("", lines[0]); + assertEquals("@duke the test group foobar does not exist", lines[1]); + + // Trying to test again should be fine + var thirdComment = new Comment("2", "/test tier1", duke, now, now); + pr.comments.add(thirdComment); + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(thirdComment, comments.get(2)); + + var fourthComment = comments.get(3); + assertEquals(bot, fourthComment.author()); + + lines = fourthComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until 01234567", + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(thirdComment, comments.get(2)); + assertEquals(fourthComment, comments.get(3)); + } + } + + @Test + void cancelAtestCommentShouldBeCancel() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of("0", Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = new Hash("01234567890123456789012345789012345789"); + + var now = ZonedDateTime.now(); + var testComment = new Comment("0", "/test tier1", duke, now, now); + var cancelComment = new Comment("1", "/test cancel", duke, now, now); + pr.comments = new ArrayList<>(List.of(testComment, cancelComment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(testComment, comments.get(0)); + assertEquals(cancelComment, comments.get(1)); + } + } + + @Test + void cancellingAPendingTestCommentShouldWork() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of(approvers, Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = new Hash("01234567890123456789012345789012345789"); + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until 01234567", + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + + // Cancelling the test now should be fine + var cancelComment = new Comment("2", "/test cancel", duke, now, now); + pr.comments.add(cancelComment); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(3, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(cancelComment, comments.get(2)); + + // Approving the test should not start a job, it has already been cancelled + var member = new HostUser(3, "foo", "Foo Bar"); + host.groups = Map.of(approvers, Set.of(member)); + var approveComment = new Comment("3", "/test approve", member, now, now); + pr.comments.add(approveComment); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(cancelComment, comments.get(2)); + assertEquals(approveComment, comments.get(3)); + } + } + + @Test + void cancellingApprovedPendingRequestShouldBeCancelled() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of(approvers, Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = new Hash("01234567890123456789012345789012345789"); + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until 01234567", + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + + // Approve the request + var member = new HostUser(2, "foo", "Foo Bar"); + host.groups = Map.of(approvers, Set.of(member)); + var approveComment = new Comment("2", "/test approve", member, now, now); + pr.comments.add(approveComment); + + // Cancelling the request + var cancelComment = new Comment("2", "/test cancel", duke, now, now); + pr.comments.add(cancelComment); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(approveComment, comments.get(2)); + assertEquals(cancelComment, comments.get(3)); + } + } + + @Test + void approvedPendingRequestShouldBeStarted() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var localRepoDir = tmp.path().resolve("repository.git"); + var localRepo = Repository.init(localRepoDir, VCS.GIT); + var readme = localRepoDir.resolve("README"); + Files.writeString(readme, "Hello\n"); + localRepo.add(readme); + var head = localRepo.commit("Add README", "duke", "duke@openjdk.org"); + + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of(approvers, Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + repo.webUrl = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.url = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.id = 1337L; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + pr.id = "17"; + pr.targetRef = "master"; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = head; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until " + head.abbreviate(), + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + + // Approve the request + var member = new HostUser(2, "foo", "Foo Bar"); + host.groups = Map.of(approvers, Set.of(member)); + var approveComment = new Comment("2", "/test approve", member, now, now); + pr.comments.add(approveComment); + + var expectedJobId = "null-1337-17-0"; + var expectedJob = new InMemoryJob(); + expectedJob.status = new Job.Status(0, 1, 7); + ci.jobs.put(expectedJobId, expectedJob); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(approveComment, comments.get(2)); + + var fourthComment = comments.get(3); + lines = fourthComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("A test job has been started with id: " + expectedJobId, lines[3]); + + assertEquals(1, ci.submissions.size()); + var submission = ci.submissions.get(0); + assertTrue(submission.source.startsWith(storage)); + assertEquals(List.of("tier1"), submission.jobs); + assertEquals(expectedJobId, submission.id); + + var checks = pr.checks(pr.headHash()); + assertEquals(1, checks.keySet().size()); + var check = checks.get("test"); + assertEquals("Summary", check.title().get()); + assertTrue(check.summary() + .get() + .contains("0 jobs completed, 1 job running, 7 jobs not yet started")); + } + } + + @Test + void cancellingApprovedPendingRequestShouldBeCancel() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var localRepoDir = tmp.path().resolve("repository.git"); + var localRepo = Repository.init(localRepoDir, VCS.GIT); + var readme = localRepoDir.resolve("README"); + Files.writeString(readme, "Hello\n"); + localRepo.add(readme); + var head = localRepo.commit("Add README", "duke", "duke@openjdk.org"); + + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of(approvers, Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + repo.webUrl = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.url = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.id = 1337L; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + pr.id = "17"; + pr.targetRef = "master"; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = head; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until " + head.abbreviate(), + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + + // Approve the request + var member = new HostUser(2, "foo", "Foo Bar"); + host.groups = Map.of(approvers, Set.of(member)); + var approveComment = new Comment("2", "/test approve", member, now, now); + pr.comments.add(approveComment); + + var expectedJobId = "null-1337-17-0"; + var expectedJob = new InMemoryJob(); + expectedJob.status = new Job.Status(0, 1, 7); + ci.jobs.put(expectedJobId, expectedJob); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(approveComment, comments.get(2)); + + var fourthComment = comments.get(3); + lines = fourthComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("A test job has been started with id: " + expectedJobId, lines[3]); + + assertEquals(1, ci.submissions.size()); + var submission = ci.submissions.get(0); + assertTrue(submission.source.startsWith(storage)); + assertEquals(List.of("tier1"), submission.jobs); + assertEquals(expectedJobId, submission.id); + + var checks = pr.checks(pr.headHash()); + assertEquals(1, checks.keySet().size()); + var check = checks.get("test"); + assertEquals("Summary", check.title().get()); + assertEquals(CheckStatus.IN_PROGRESS, check.status()); + assertTrue(check.summary() + .get() + .contains("## Status\n0 jobs completed, 1 job running, 7 jobs not yet started\n")); + + var cancelComment = new Comment("4", "/test cancel", duke, now, now); + pr.comments.add(cancelComment); + + item.run(scratch); + + checks = pr.checks(pr.headHash()); + assertEquals(1, checks.keySet().size()); + check = checks.get("test"); + assertEquals("Summary", check.title().get()); + assertEquals(CheckStatus.CANCELLED, check.status()); + assertTrue(check.summary() + .get() + .contains("## Status\n0 jobs completed, 1 job running, 7 jobs not yet started\n")); + + assertEquals(expectedJobId, ci.cancelled.get(0)); + } + } + + @Test + void errorWhenCreatingTestJobShouldResultInError() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var localRepoDir = tmp.path().resolve("repository.git"); + var localRepo = Repository.init(localRepoDir, VCS.GIT); + var readme = localRepoDir.resolve("README"); + Files.writeString(readme, "Hello\n"); + localRepo.add(readme); + var head = localRepo.commit("Add README", "duke", "duke@openjdk.org"); + + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of(approvers, Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + repo.webUrl = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.url = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.id = 1337L; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + pr.id = "17"; + pr.targetRef = "master"; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = head; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until " + head.abbreviate(), + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + + // Approve the request + var member = new HostUser(2, "foo", "Foo Bar"); + host.groups = Map.of(approvers, Set.of(member)); + var approveComment = new Comment("2", "/test approve", member, now, now); + pr.comments.add(approveComment); + + ci.throwOnSubmit = true; + assertThrows(UncheckedIOException.class, () -> item.run(scratch)); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(approveComment, comments.get(2)); + + var fifthComment = comments.get(3); + lines = fifthComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("Could not create test job", lines[1]); + } + } + + @Test + void finishedJobShouldResultInFinishedComment() throws IOException { + try (var tmp = new TemporaryDirectory()) { + var localRepoDir = tmp.path().resolve("repository.git"); + var localRepo = Repository.init(localRepoDir, VCS.GIT); + var readme = localRepoDir.resolve("README"); + Files.writeString(readme, "Hello\n"); + localRepo.add(readme); + var head = localRepo.commit("Add README", "duke", "duke@openjdk.org"); + + var ci = new InMemoryContinuousIntegration(); + var approvers = "0"; + var available = List.of("tier1", "tier2", "tier3"); + var defaultJobs = List.of("tier1"); + var name = "test"; + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("storage"); + + var bot = new HostUser(1, "bot", "openjdk [bot]"); + var host = new InMemoryHost(); + host.currentUserDetails = bot; + host.groups = Map.of(approvers, Set.of()); + + var repo = new InMemoryHostedRepository(); + repo.host = host; + repo.webUrl = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.url = URI.create("file://" + localRepoDir.toAbsolutePath()); + repo.id = 1337L; + + var pr = new InMemoryPullRequest(); + pr.repository = repo; + pr.id = "17"; + pr.targetRef = "master"; + + var duke = new HostUser(0, "duke", "Duke"); + pr.author = duke; + pr.headHash = head; + + var now = ZonedDateTime.now(); + var comment = new Comment("0", "/test tier1", duke, now, now); + pr.comments = new ArrayList<>(List.of(comment)); + + var item = new TestWorkItem(ci, approvers, available, defaultJobs, name, storage, pr); + + item.run(scratch); + + var comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + var secondComment = comments.get(1); + assertEquals(bot, secondComment.author()); + + var lines = secondComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke you need to get approval to run the tests in tier1 for commits up until " + head.abbreviate(), + lines[3]); + + // Nothing should change if we run it yet again + item.run(scratch); + + comments = pr.comments(); + assertEquals(2, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + + // Approve the request + var member = new HostUser(2, "foo", "Foo Bar"); + host.groups = Map.of(approvers, Set.of(member)); + var approveComment = new Comment("2", "/test approve", member, now, now); + pr.comments.add(approveComment); + + var expectedJobId = "null-1337-17-0"; + var expectedJob = new InMemoryJob(); + expectedJob.status = new Job.Status(0, 1, 7); + ci.jobs.put(expectedJobId, expectedJob); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(4, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(approveComment, comments.get(2)); + + var fourthComment = comments.get(3); + lines = fourthComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("A test job has been started with id: " + expectedJobId, lines[3]); + + assertEquals(1, ci.submissions.size()); + var submission = ci.submissions.get(0); + assertTrue(submission.source.startsWith(storage)); + assertEquals(List.of("tier1"), submission.jobs); + assertEquals(expectedJobId, submission.id); + + var checks = pr.checks(pr.headHash()); + assertEquals(1, checks.keySet().size()); + var check = checks.get("test"); + assertEquals("Summary", check.title().get()); + assertEquals(CheckStatus.IN_PROGRESS, check.status()); + assertTrue(check.summary() + .get() + .contains("0 jobs completed, 1 job running, 7 jobs not yet started")); + + var job = ci.jobs.get(expectedJobId); + assertNotNull(job); + job.id = "id"; + job.state = Job.State.COMPLETED; + job.status = new Job.Status(8, 0, 0); + job.result = new Job.Result(8, 0, 0); + + item.run(scratch); + + comments = pr.comments(); + assertEquals(5, comments.size()); + assertEquals(comment, comments.get(0)); + assertEquals(secondComment, comments.get(1)); + assertEquals(approveComment, comments.get(2)); + assertEquals(fourthComment, comments.get(3)); + + var finishedComment = comments.get(4); + lines = finishedComment.body().split("\n"); + assertEquals("", lines[0]); + assertEquals("", lines[1]); + assertEquals("", lines[2]); + assertEquals("@duke your test job with id " + expectedJobId + " for commits up until " + + head.abbreviate() + " has finished.", lines[3]); + + checks = pr.checks(pr.headHash()); + assertEquals(1, checks.keySet().size()); + check = checks.get("test"); + assertEquals("Summary", check.title().get()); + assertEquals(CheckStatus.SUCCESS, check.status()); + + var summaryLines = check.summary().get().split("\n"); + assertEquals("## Id", summaryLines[0]); + assertEquals("`id`", summaryLines[1]); + assertEquals("", summaryLines[2]); + assertEquals("## Builds", summaryLines[3]); + assertEquals("", summaryLines[4]); + assertEquals("## Tests", summaryLines[5]); + assertEquals("", summaryLines[6]); + assertEquals("## Status", summaryLines[7]); + assertEquals("8 jobs completed, 0 jobs running, 0 jobs not yet started", summaryLines[8]); + assertEquals("", summaryLines[9]); + assertEquals("## Result", summaryLines[10]); + assertEquals("8 jobs passed, 0 jobs with failures, 0 jobs not run", summaryLines[11]); + } + } +} diff --git a/bots/topological/build.gradle b/bots/topological/build.gradle index b7b7e36bc..bff8fb6c7 100644 --- a/bots/topological/build.gradle +++ b/bots/topological/build.gradle @@ -32,6 +32,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':host') implementation project(':forge') implementation project(':issuetracker') diff --git a/build.gradle b/build.gradle index 350d23f6b..dbef0890f 100644 --- a/build.gradle +++ b/build.gradle @@ -104,19 +104,82 @@ reproduce { dockerfile = 'test.dockerfile' } +def getOS() { + def os = System.getProperty('os.name').toLowerCase() + if (os.startsWith('linux')) { + return 'linux' + } + if (os.startsWith('mac')) { + return 'macos' + } + if (os.startsWith('win')) { + return 'windows' + } + if (os.startsWith('sunos')) { + return 'solaris' + } + throw new GradleException("Unexpected operating system: " + os) +} + +def getCPU() { + def cpu = System.getProperty('os.arch').toLowerCase() + if (cpu.startsWith('amd64') || cpu.startsWith('x86_64') || cpu.startsWith('x64')) { + return 'x64' + } + if (cpu.startsWith('x86') || cpu.startsWith('i386')) { + return 'x86' + } + if (cpu.startsWith('sparc')) { + return 'sparc' + } + if (cpu.startsWith('ppc')) { + return 'ppc' + } + if (cpu.startsWith('arm')) { + return 'arm' + } + if (cpu.startsWith('aarch64')) { + return 'aarch64'; + } + throw new GradleException("Unexpected CPU: " + cpu) +} + task local(type: Copy) { doFirst { delete project.buildDir } - def os = System.getProperty('os.name').toLowerCase() - def osName = os.startsWith('win') ? 'Windows' : - os.startsWith('mac') ? 'Macos' : 'Linux' - dependsOn ':cli:image' + osName + def os = getOS() + def cpu = getCPU() + + if (os in ['linux', 'macos', 'windows'] && cpu == 'x64') { + def target = os.substring(0, 1).toUpperCase() + os.substring(1) + + cpu.substring(0, 1).toUpperCase() + cpu.substring(1) + dependsOn ':cli:image' + target + } else { + dependsOn ':cli:imageLocal' + } + + from zipTree(file(project.rootDir.toString() + + '/cli/build/distributions/cli' + + '-' + project.version + '-' + + os + '-' + cpu + '.zip')) + into project.buildDir +} + +task offline(type: Copy) { + doFirst { + delete project.buildDir + } + + def os = getOS() + def cpu = getCPU() + + dependsOn ':cli:imageLocal' from zipTree(file(project.rootDir.toString() + '/cli/build/distributions/cli' + '-' + project.version + '-' + - osName.toLowerCase() + '.zip')) + os + '-' + cpu + '.zip')) into project.buildDir } diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 493965582..b2f2bca71 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -26,7 +26,7 @@ plugins { } dependencies { - runtime subprojects.collect { owner.project(it.path) } + runtimeOnly subprojects.collect { owner.project(it.path) } } defaultTasks 'compileJava' diff --git a/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/ImagesPlugin.java b/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/ImagesPlugin.java index dbb808be7..01b9c2a69 100644 --- a/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/ImagesPlugin.java +++ b/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/ImagesPlugin.java @@ -29,9 +29,52 @@ import org.gradle.api.artifacts.UnknownConfigurationException; import java.util.ArrayList; +import java.util.HashSet; import java.io.File; public class ImagesPlugin implements Plugin { + private static String getOS() { + var p = System.getProperty("os.name").toLowerCase(); + if (p.startsWith("win")) { + return "windows"; + } + if (p.startsWith("mac")) { + return "macos"; + } + if (p.startsWith("linux")) { + return "linux"; + } + if (p.startsWith("sunos")) { + return "solaris"; + } + + throw new RuntimeException("Unknown operating system: " + System.getProperty("os.name")); + } + + private static String getCPU() { + var p = System.getProperty("os.arch").toLowerCase(); + if (p.startsWith("amd64") || p.startsWith("x86_64") || p.startsWith("x64")) { + return "x64"; + } + if (p.startsWith("x86") || p.startsWith("i386")) { + return "x86"; + } + if (p.startsWith("sparc")) { + return "sparc"; + } + if (p.startsWith("ppc")) { + return "ppc"; + } + if (p.startsWith("arm")) { + return "arm"; + } + if (p.startsWith("aarch64")) { + return "aarch64"; + } + + throw new RuntimeException("Unknown CPU: " + System.getProperty("os.arch")); + } + @Override public void apply(Project project) { NamedDomainObjectContainer imageEnvironmentContainer = @@ -49,15 +92,23 @@ public ImageEnvironment create(String name) { imageEnvironmentContainer.all(new Action() { public void execute(ImageEnvironment env) { - var name = env.getName(); - var subName = name.substring(0, 1).toUpperCase() + name.substring(1); + var parts = env.getName().split("_");; + var isLocal = parts.length == 1 && parts[0].equals("local"); + var os = isLocal ? getOS() : parts[0]; + var cpu = isLocal ? getCPU() : parts[1]; + var osAndCpuPascalCased = + os.substring(0, 1).toUpperCase() + os.substring(1) + + cpu.substring(0, 1).toUpperCase() + cpu.substring(1); + var subName = isLocal ? "Local" : osAndCpuPascalCased; var downloadTaskName = "download" + subName + "JDK"; - project.getTasks().register(downloadTaskName, DownloadJDKTask.class, (task) -> { - task.getUrl().set(env.getUrl()); - task.getSha256().set(env.getSha256()); - task.getToDir().set(rootDir.resolve(".jdk")); - }); + if (!isLocal) { + project.getTasks().register(downloadTaskName, DownloadJDKTask.class, (task) -> { + task.getUrl().set(env.getUrl()); + task.getSha256().set(env.getSha256()); + task.getToDir().set(rootDir.resolve(".jdk")); + }); + } var linkTaskName = "link" + subName; project.getTasks().register(linkTaskName, LinkTask.class, (task) -> { @@ -75,10 +126,15 @@ public void execute(ImageEnvironment env) { // ignored } - task.dependsOn(projectPath + ":" + downloadTaskName); + if (!isLocal) { + task.dependsOn(projectPath + ":" + downloadTaskName); + task.getUrl().set(env.getUrl()); + } else { + task.getUrl().set("local"); + } task.getToDir().set(buildDir.resolve("images")); - task.getUrl().set(env.getUrl()); - task.getOS().set(name); + task.getOS().set(os); + task.getCPU().set(cpu); task.getLaunchers().set(env.getLaunchers()); task.getModules().set(env.getModules()); }); @@ -88,7 +144,8 @@ public void execute(ImageEnvironment env) { task.getLaunchers().set(env.getLaunchers()); task.getOptions().set(env.getOptions()); task.getToDir().set(buildDir.resolve("launchers")); - task.getOS().set(name); + task.getOS().set(os); + task.getCPU().set(cpu); }); var zipTaskName = "bundleZip" + subName; @@ -99,7 +156,7 @@ public void execute(ImageEnvironment env) { task.setPreserveFileTimestamps(false); task.setReproducibleFileOrder(true); task.getArchiveBaseName().set(project.getName()); - task.getArchiveClassifier().set(name); + task.getArchiveClassifier().set(os + "-" + cpu); task.getArchiveExtension().set("zip"); if (env.getMan().isPresent()) { @@ -109,10 +166,11 @@ public void execute(ImageEnvironment env) { }); } - task.from(buildDir.resolve("images").resolve(name), (s) -> { + var subdir = os + "-" + cpu; + task.from(buildDir.resolve("images").resolve(subdir), (s) -> { s.into("image"); }); - task.from(buildDir.resolve("launchers").resolve(name), (s) -> { + task.from(buildDir.resolve("launchers").resolve(subdir), (s) -> { s.into("bin"); }); }); @@ -125,7 +183,7 @@ public void execute(ImageEnvironment env) { task.setPreserveFileTimestamps(false); task.setReproducibleFileOrder(true); task.getArchiveBaseName().set(project.getName()); - task.getArchiveClassifier().set(name); + task.getArchiveClassifier().set(os + "-" + cpu); task.getArchiveExtension().set("tar.gz"); task.setCompression(Compression.GZIP); @@ -136,10 +194,11 @@ public void execute(ImageEnvironment env) { }); } - task.from(buildDir.resolve("images").resolve(name), (s) -> { + var subdir = os + "-" + cpu; + task.from(buildDir.resolve("images").resolve(subdir), (s) -> { s.into("image"); }); - task.from(buildDir.resolve("launchers").resolve(name), (s) -> { + task.from(buildDir.resolve("launchers").resolve(subdir), (s) -> { s.into("bin"); }); }); diff --git a/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LaunchersTask.java b/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LaunchersTask.java index 59f13e2da..85339669b 100644 --- a/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LaunchersTask.java +++ b/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LaunchersTask.java @@ -37,6 +37,7 @@ public class LaunchersTask extends DefaultTask { private Property toDir; private Property os; + private Property cpu; private MapProperty launchers; private ListProperty options; @@ -44,6 +45,7 @@ public class LaunchersTask extends DefaultTask { public LaunchersTask(ObjectFactory factory) { toDir = factory.property(Path.class); os = factory.property(String.class); + cpu = factory.property(String.class); launchers = factory.mapProperty(String.class, String.class); options = factory.listProperty(String.class); } @@ -63,6 +65,11 @@ Property getOS() { return os; } + @Input + Property getCPU() { + return cpu; + } + @Input MapProperty getLaunchers() { return launchers; @@ -77,7 +84,7 @@ private static void clearDirectory(Path directory) throws IOException { @TaskAction void generate() throws IOException { - var dest = toDir.get().resolve(os.get()); + var dest = toDir.get().resolve(os.get() + "-" + cpu.get()); if (Files.isDirectory(dest)) { clearDirectory(dest); } diff --git a/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LinkTask.java b/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LinkTask.java index 6563e8db8..1cd6a8b7a 100644 --- a/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LinkTask.java +++ b/buildSrc/images/src/main/java/org/openjdk/skara/gradle/images/LinkTask.java @@ -41,6 +41,7 @@ public class LinkTask extends DefaultTask { private final Property os; + private final Property cpu; private final Property url; private final Property toDir; private final MapProperty launchers; @@ -51,6 +52,7 @@ public class LinkTask extends DefaultTask { @Inject public LinkTask(ObjectFactory factory) { os = factory.property(String.class); + cpu = factory.property(String.class); url = factory.property(String.class); toDir = factory.property(Path.class); launchers = factory.mapProperty(String.class, String.class); @@ -69,6 +71,11 @@ Property getOS() { return os; } + @Input + Property getCPU() { + return cpu; + } + @Input Property getUrl() { return url; @@ -117,9 +124,14 @@ void link() throws IOException { modularJars.add(jar.getAsFile().toString()); } - var filename = Path.of(URI.create(url.get()).getPath()).getFileName().toString(); - var dirname = filename.replace(".zip", "").replace(".tar.gz", ""); - var jdk = project.getRootDir().toPath().toAbsolutePath().resolve(".jdk").resolve(dirname); + Path jdk = null; + if (!url.get().equals("local")) { + var filename = Path.of(URI.create(url.get()).getPath()).getFileName().toString(); + var dirname = filename.replace(".zip", "").replace(".tar.gz", ""); + jdk = project.getRootDir().toPath().toAbsolutePath().resolve(".jdk").resolve(dirname); + } else { + jdk = Path.of(System.getProperty("java.home")); + } var dirs = Files.walk(jdk) .filter(Files::isDirectory) .filter(p -> p.getFileName().toString().equals("jmods")) @@ -143,7 +155,7 @@ void link() throws IOException { var allModules = new ArrayList(uniqueModules); Files.createDirectories(toDir.get()); - var dest = toDir.get().resolve(os.get()); + var dest = toDir.get().resolve(os.get() + "-" + cpu.get()); if (Files.exists(dest) && Files.isDirectory(dest)) { clearDirectory(dest); } @@ -163,14 +175,14 @@ void link() throws IOException { }); var currentOS = System.getProperty("os.name").toLowerCase().substring(0, 3); - if (currentOS.equals(os.get().substring(0, 3))) { + if (os.get().equals("local") || currentOS.equals(os.get().substring(0, 3))) { var ext = currentOS.startsWith("win") ? ".exe" : ""; var javaLaunchers = Files.walk(dest) .filter(Files::isExecutable) .filter(p -> p.getFileName().toString().equals("java" + ext)) .collect(Collectors.toList()); if (javaLaunchers.size() != 1) { - throw new GradleException("Multiple or no java launchers generated for " + os.get() + " image"); + throw new GradleException("Multiple or no java launchers generated for " + os.get() + "-" + cpu.get() + " image"); } var java = javaLaunchers.get(0); project.exec((spec) -> { diff --git a/ci/build.gradle b/ci/build.gradle new file mode 100644 index 000000000..1c4afd9e2 --- /dev/null +++ b/ci/build.gradle @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +module { + name = 'org.openjdk.skara.ci' + test { + requires 'org.junit.jupiter.api' + requires 'org.openjdk.skara.test' + opens 'org.openjdk.skara.ci' to 'org.junit.platform.commons' + } +} + +dependencies { + implementation project(':host') + implementation project(':json') +} diff --git a/ci/src/main/java/module-info.java b/ci/src/main/java/module-info.java new file mode 100644 index 000000000..3247e493a --- /dev/null +++ b/ci/src/main/java/module-info.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +module org.openjdk.skara.ci { + requires org.openjdk.skara.host; + requires org.openjdk.skara.json; + + uses org.openjdk.skara.ci.ContinuousIntegrationFactory; + exports org.openjdk.skara.ci; +} diff --git a/ci/src/main/java/org/openjdk/skara/ci/Build.java b/ci/src/main/java/org/openjdk/skara/ci/Build.java new file mode 100644 index 000000000..32910c64b --- /dev/null +++ b/ci/src/main/java/org/openjdk/skara/ci/Build.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.ci; + +import java.util.Objects; + +public class Build { + public static enum OperatingSystem { + WINDOWS, + MACOS, + LINUX, + SOLARIS, + AIX, + FREEBSD, + OPENBSD, + NETBSD, + HPUX, + HAIKU + } + + public static enum CPU { + X86, + X64, + SPARCV9, + AARCH64, + AARCH32, + PPCLE32, + PPCLE64 + } + + public static enum DebugLevel { + RELEASE, + FASTDEBUG, + SLOWDEBUG + } + + private final OperatingSystem os; + private final CPU cpu; + private final DebugLevel debugLevel; + + public Build(OperatingSystem os, CPU cpu, DebugLevel debugLevel) { + this.os = os; + this.cpu = cpu; + this.debugLevel = debugLevel; + } + + public OperatingSystem os() { + return os; + } + + public CPU cpu() { + return cpu; + } + + public DebugLevel debugLevel() { + return debugLevel; + } + + @Override + public String toString() { + return os.toString().toLowerCase() + "-" + + cpu.toString().toLowerCase() + "-" + + debugLevel.toString().toLowerCase(); + } + + @Override + public int hashCode() { + return Objects.hash(os, cpu, debugLevel); + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + + if (!(other instanceof Build)) { + return false; + } + + var o = (Build) other; + return Objects.equals(os, o.os) && + Objects.equals(cpu, o.cpu) && + Objects.equals(debugLevel, o.debugLevel); + } +} diff --git a/ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegration.java b/ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegration.java new file mode 100644 index 000000000..9ba03243b --- /dev/null +++ b/ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegration.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.ci; + +import org.openjdk.skara.host.Host; +import org.openjdk.skara.json.JSONObject; +import org.openjdk.skara.json.JSON; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Path; +import java.util.*; + +public interface ContinuousIntegration extends Host { + Job submit(Path source, List jobs, String id) throws IOException; + Job job(String id) throws IOException; + List query(String query) throws IOException; + void cancel(String id) throws IOException; + + static Optional from(URI uri, JSONObject configuration) { + for (var factory : ContinuousIntegrationFactory.factories()) { + var ci = factory.create(uri, configuration); + if (ci.isValid()) { + return Optional.of(ci); + } + } + return Optional.empty(); + } + + static Optional from(URI uri) { + return from(uri, JSON.object()); + } +} diff --git a/ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegrationFactory.java b/ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegrationFactory.java new file mode 100644 index 000000000..1d5f0dcbf --- /dev/null +++ b/ci/src/main/java/org/openjdk/skara/ci/ContinuousIntegrationFactory.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.ci; + +import org.openjdk.skara.json.JSONObject; + +import java.net.URI; +import java.util.*; +import java.util.stream.*; + +public interface ContinuousIntegrationFactory { + ContinuousIntegration create(URI uri, JSONObject configuration); + + static List factories() { + return StreamSupport.stream(ServiceLoader.load(ContinuousIntegrationFactory.class).spliterator(), false) + .collect(Collectors.toList()); + } +} diff --git a/ci/src/main/java/org/openjdk/skara/ci/Job.java b/ci/src/main/java/org/openjdk/skara/ci/Job.java new file mode 100644 index 000000000..ad510a698 --- /dev/null +++ b/ci/src/main/java/org/openjdk/skara/ci/Job.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.ci; + +import java.util.List; + +public interface Job { + static class Status { + private final int numCompleted; + private final int numRunning; + private final int numNotStarted; + + public Status(int numCompleted, int numRunning, int numNotStarted) { + this.numCompleted = numCompleted; + this.numRunning = numRunning; + this.numNotStarted = numNotStarted; + } + + public int numCompleted() { + return numCompleted; + } + + public int numRunning() { + return numRunning; + } + + public int numNotStarted() { + return numNotStarted; + } + + public int numTotal() { + return numCompleted + numRunning + numNotStarted; + } + } + + static class Result { + private final int numPassed; + private final int numFailed; + private final int numSkipped; + + public Result(int numPassed, int numFailed, int numSkipped) { + this.numPassed = numPassed; + this.numFailed = numFailed; + this.numSkipped = numSkipped; + } + + public int numPassed() { + return numPassed; + } + + public int numFailed() { + return numFailed; + } + + public int numSkipped() { + return numSkipped; + } + + public int numTotal() { + return numPassed + numFailed + numSkipped; + } + } + + String id(); + List builds(); + List tests(); + Status status(); + Result result(); + + static enum State { + COMPLETED, + RUNNING, + SCHEDULED + } + State state(); + default boolean isCompleted() { + return state() == State.COMPLETED; + } + default boolean isRunning() { + return state() == State.COMPLETED; + } + default boolean isScheduled() { + return state() == State.SCHEDULED; + } +} diff --git a/ci/src/main/java/org/openjdk/skara/ci/Test.java b/ci/src/main/java/org/openjdk/skara/ci/Test.java new file mode 100644 index 000000000..23ec682e7 --- /dev/null +++ b/ci/src/main/java/org/openjdk/skara/ci/Test.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.ci; + +public class Test { + public static enum Kind { + SINGLE, + GROUP + } + + private final Kind kind; + private final String name; + + public Test(Kind kind, String name) { + this.kind = kind; + this.name = name; + } + + public Kind kind() { + return kind; + } + + public String name() { + return name; + } +} diff --git a/cli/build.gradle b/cli/build.gradle index 9a9b4c3ac..ba885a4c4 100644 --- a/cli/build.gradle +++ b/cli/build.gradle @@ -47,7 +47,7 @@ dependencies { jar { manifest { - attributes("Implementation-Title": "org.openjdk.skara.cli", "Implementation-Version": version) + attributes("Implementation-Title": "org.openjdk.skara.cli", "Implementation-Version": archiveVersion) } } @@ -71,35 +71,42 @@ images { ext.modules = ['jdk.crypto.ec'] - windows { + windows_x64 { modules = ext.modules launchers = ext.launchers bundles = ['zip', 'tar.gz'] jdk { - url = 'https://download.java.net/java/GA/jdk12/GPL/openjdk-12_windows-x64_bin.zip' - sha256 = '35a8d018f420fb05fe7c2aa9933122896ca50bd23dbd373e90d8e2f3897c4e92' + url = 'https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_windows-x64_bin.zip' + sha256 = '438a6920f1851b1eeb6f09f05d9f91c4423c6586f7a1a7ccbb19df76ea5901ee' } } - linux { + linux_x64 { modules = ext.modules launchers = ext.launchers man = 'cli/resources/man' bundles = ['zip', 'tar.gz'] jdk { - url = 'https://download.java.net/java/GA/jdk12/GPL/openjdk-12_linux-x64_bin.tar.gz' - sha256 = 'b43bc15f4934f6d321170419f2c24451486bc848a2179af5e49d10721438dd56' + url = 'https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_linux-x64_bin.tar.gz' + sha256 = '2e01716546395694d3fad54c9b36d1cd46c5894c06f72d156772efbcf4b41335' } } - macos { + macos_x64 { modules = ext.modules launchers = ext.launchers man = 'cli/resources/man' bundles = ['zip', 'tar.gz'] jdk { - url = 'https://download.java.net/java/GA/jdk12/GPL/openjdk-12_osx-x64_bin.tar.gz' - sha256 = '52164a04db4d3fdfe128cfc7b868bc4dae52d969f03d53ae9d4239fe783e1a3a' + url = 'https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_osx-x64_bin.tar.gz' + sha256 = '593c5c9dc0978db21b06d6219dc8584b76a59c79d57e6ec1b28ad0d848a7713f' } } + + local { + modules = ext.modules + launchers = ext.launchers + man = 'cli/resources/man' + bundles = ['zip', 'tar.gz'] + } } diff --git a/cli/src/main/java/org/openjdk/skara/cli/GitFork.java b/cli/src/main/java/org/openjdk/skara/cli/GitFork.java index 0b8c02c43..27f965430 100644 --- a/cli/src/main/java/org/openjdk/skara/cli/GitFork.java +++ b/cli/src/main/java/org/openjdk/skara/cli/GitFork.java @@ -146,7 +146,10 @@ public static void main(String[] args) throws IOException { exit("No username for host " + hostName + " found, use git-credentials or the flag --username"); } - var host = Forge.from(uri, new PersonalAccessToken(credentials.username(), credentials.password())); + var host = Forge.from(uri, new Credential(credentials.username(), credentials.password())); + if (host.isEmpty() || !host.get().isValid()) { + exit("Failed to connect to host " + hostName); + } if (path.endsWith(".git")) { path = path.substring(0, path.length() - 4); } @@ -154,7 +157,10 @@ public static void main(String[] args) throws IOException { path = path.substring(1); } - var fork = host.repository(path).fork(); + var hostedRepo = host.get().repository(path).orElseThrow(() -> + new IOException("Could not find repository at " + uri.toString()) + ); + var fork = hostedRepo.fork(); if (token == null) { GitCredentials.approve(credentials); diff --git a/cli/src/main/java/org/openjdk/skara/cli/GitPr.java b/cli/src/main/java/org/openjdk/skara/cli/GitPr.java index b09ebdc0f..c797278ed 100644 --- a/cli/src/main/java/org/openjdk/skara/cli/GitPr.java +++ b/cli/src/main/java/org/openjdk/skara/cli/GitPr.java @@ -100,23 +100,21 @@ private static String projectName(URI uri) { return name; } - private static HostedRepository getHostedRepositoryFor(URI uri, GitCredentials credentials) throws IOException { - var host = Forge.from(uri, new PersonalAccessToken(credentials.username(), credentials.password())); - if (System.getenv("GIT_TOKEN") == null) { - GitCredentials.approve(credentials); - } - var remoteRepo = host.repository(projectName(uri)); + private static HostedRepository getHostedRepositoryFor(URI uri, Forge host) throws IOException { + var remoteRepo = host.repository(projectName(uri)).orElseThrow(() -> + new IOException("Could not find repository at: " + uri.toString()) + ); var parentRepo = remoteRepo.parent(); var targetRepo = parentRepo.isPresent() ? parentRepo.get() : remoteRepo; return targetRepo; } - private static PullRequest getPullRequest(URI uri, GitCredentials credentials, Argument prId) throws IOException { + private static PullRequest getPullRequest(URI uri, Forge host, Argument prId) throws IOException { if (!prId.isPresent()) { exit("error: missing pull request identifier"); } - var pr = getHostedRepositoryFor(uri, credentials).pullRequest(prId.asString()); + var pr = getHostedRepositoryFor(uri, host).pullRequest(prId.asString()); if (pr == null) { exit("error: could not fetch PR information"); } @@ -257,6 +255,10 @@ public static void main(String[] args) throws IOException, InterruptedException .fullname("no-decoration") .helptext("Hide any decorations when listing PRs") .optional(), + Switch.shortcut("") + .fullname("no-token") + .helptext("Do not use a personal access token (PAT). Only works for read-only operations.") + .optional(), Switch.shortcut("") .fullname("mercurial") .helptext("Force use of Mercurial (hg)") @@ -308,10 +310,36 @@ public static void main(String[] args) throws IOException, InterruptedException var username = arguments.contains("username") ? arguments.get("username").asString() : null; var token = isMercurial ? System.getenv("HG_TOKEN") : System.getenv("GIT_TOKEN"); var uri = Remote.toWebURI(remotePullPath); - var credentials = GitCredentials.fill(uri.getHost(), uri.getPath(), username, token, uri.getScheme()); - var host = Forge.from(uri, new PersonalAccessToken(credentials.username(), credentials.password())); + var shouldUseToken = !arguments.contains("no-token"); + var credentials = !shouldUseToken ? + null : + GitCredentials.fill(uri.getHost(), uri.getPath(), username, token, uri.getScheme()); + var forgeURI = URI.create(uri.getScheme() + "://" + uri.getHost()); + var forge = credentials == null ? + Forge.from(forgeURI) : + Forge.from(forgeURI, new Credential(credentials.username(), credentials.password())); + if (forge.isEmpty() || !forge.get().isValid()) { + if (!shouldUseToken) { + if (arguments.contains("verbose")) { + System.err.println(""); + } + System.err.println("warning: using git-pr with --no-token may result in rate limiting from " + forgeURI); + if (!arguments.contains("verbose")) { + System.err.println(" Re-run git-pr with --verbose to see if you are being rate limited"); + System.err.println(""); + } + } + exit("error: failed to connect to host: " + forgeURI); + } + var host = forge.get(); var action = arguments.at(0).asString(); + if (!shouldUseToken && + !List.of("list", "fetch", "show", "checkout", "apply").contains(action)) { + System.err.println("error: --no-token can only be used with read-only operations"); + System.exit(1); + } + if (action.equals("create")) { if (isMercurial) { var currentBookmark = repo.currentBookmark(); @@ -412,7 +440,9 @@ public static void main(String[] args) throws IOException, InterruptedException System.exit(1); } - var remoteRepo = host.repository(projectName(uri)); + var remoteRepo = host.repository(projectName(uri)).orElseThrow(() -> + new IOException("Could not find repository at " + uri.toString()) + ); if (token == null) { GitCredentials.approve(credentials); } @@ -480,7 +510,7 @@ public static void main(String[] args) throws IOException, InterruptedException if (arguments.contains("assignees")) { var usernames = Arrays.asList(arguments.get("assignees").asString().split(",")); var assignees = usernames.stream() - .map(host::user) + .map(u -> host.user(u)) .collect(Collectors.toList()); pr.setAssignees(assignees); } @@ -569,7 +599,9 @@ public static void main(String[] args) throws IOException, InterruptedException System.exit(1); } - var remoteRepo = host.repository(projectName(uri)); + var remoteRepo = host.repository(projectName(uri)).orElseThrow(() -> + new IOException("Could not find repository at " + uri.toString()) + ); if (token == null) { GitCredentials.approve(credentials); } @@ -637,14 +669,14 @@ public static void main(String[] args) throws IOException, InterruptedException if (arguments.contains("assignees")) { var usernames = Arrays.asList(arguments.get("assignees").asString().split(",")); var assignees = usernames.stream() - .map(host::user) + .map(u -> host.user(u)) .collect(Collectors.toList()); pr.setAssignees(assignees); } System.out.println(pr.webUrl().toString()); Files.deleteIfExists(file); } else if (action.equals("integrate") || action.equals("approve")) { - var pr = getPullRequest(uri, credentials, arguments.at(1)); + var pr = getPullRequest(uri, host, arguments.at(1)); if (action.equals("integrate")) { pr.addComment("/integrate"); @@ -654,7 +686,7 @@ public static void main(String[] args) throws IOException, InterruptedException throw new IllegalStateException("unexpected action: " + action); } } else if (action.equals("list")) { - var remoteRepo = getHostedRepositoryFor(uri, credentials); + var remoteRepo = getHostedRepositoryFor(uri, host); var prs = remoteRepo.pullRequests(); var ids = new ArrayList(); @@ -746,7 +778,7 @@ public static void main(String[] args) throws IOException, InterruptedException exit("error: missing pull request identifier"); } - var remoteRepo = getHostedRepositoryFor(uri, credentials); + var remoteRepo = getHostedRepositoryFor(uri, host); var pr = remoteRepo.pullRequest(prId.asString()); var repoUrl = remoteRepo.webUrl(); var prHeadRef = pr.sourceRef(); @@ -820,7 +852,7 @@ public static void main(String[] args) throws IOException, InterruptedException exit("error: missing pull request identifier"); } - var remoteRepo = getHostedRepositoryFor(uri, credentials); + var remoteRepo = getHostedRepositoryFor(uri, host); var pr = remoteRepo.pullRequest(prId.asString()); pr.setState(PullRequest.State.CLOSED); } else if (action.equals("update")) { @@ -829,12 +861,12 @@ public static void main(String[] args) throws IOException, InterruptedException exit("error: missing pull request identifier"); } - var remoteRepo = getHostedRepositoryFor(uri, credentials); + var remoteRepo = getHostedRepositoryFor(uri, host); var pr = remoteRepo.pullRequest(prId.asString()); if (arguments.contains("assignees")) { var usernames = Arrays.asList(arguments.get("assignees").asString().split(",")); var assignees = usernames.stream() - .map(host::user) + .map(u -> host.user(u)) .collect(Collectors.toList()); pr.setAssignees(assignees); } diff --git a/cli/src/main/java/org/openjdk/skara/cli/GitSync.java b/cli/src/main/java/org/openjdk/skara/cli/GitSync.java index 767b248d5..1f0c732f5 100644 --- a/cli/src/main/java/org/openjdk/skara/cli/GitSync.java +++ b/cli/src/main/java/org/openjdk/skara/cli/GitSync.java @@ -24,9 +24,11 @@ import org.openjdk.skara.args.*; import org.openjdk.skara.vcs.*; +import org.openjdk.skara.forge.*; +import org.openjdk.skara.proxy.HttpProxy; import java.io.*; -import java.net.URI; +import java.net.*; import java.nio.file.*; import java.util.*; import java.util.logging.*; @@ -38,12 +40,6 @@ private static IOException die(String message) { return new IOException("will never reach here"); } - private static int fetch() throws IOException, InterruptedException { - var pb = new ProcessBuilder("git", "fetch"); - pb.inheritIO(); - return pb.start().waitFor(); - } - private static int pull() throws IOException, InterruptedException { var pb = new ProcessBuilder("git", "pull"); pb.inheritIO(); @@ -71,10 +67,6 @@ public static void main(String[] args) throws IOException, InterruptedException .fullname("pull") .helptext("Pull current branch from origin after successful sync") .optional(), - Switch.shortcut("") - .fullname("fetch") - .helptext("Fetch current branch from origin after successful sync") - .optional(), Switch.shortcut("m") .fullname("mercurial") .helptext("Force use of mercurial") @@ -111,6 +103,8 @@ public static void main(String[] args) throws IOException, InterruptedException die("error: no repository found at " + cwd.toString()) ); + HttpProxy.setup(); + var remotes = repo.remotes(); String upstream = null; @@ -121,9 +115,25 @@ public static void main(String[] args) throws IOException, InterruptedException if (lines.size() == 1 && remotes.contains(lines.get(0))) { upstream = lines.get(0); } else { - die("No remote provided to fetch from, please set the --from flag"); + if (remotes.contains("origin")) { + var originPullPath = repo.pullPath("origin"); + try { + var uri = Remote.toWebURI(originPullPath); + upstream = Forge.from(URI.create(uri.getScheme() + "://" + uri.getHost())) + .flatMap(f -> f.repository(uri.getPath().substring(1))) + .flatMap(r -> r.parent()) + .map(p -> p.webUrl().toString()) + .orElse(null); + } catch (IllegalArgumentException e) { + upstream = null; + } + } } } + + if (upstream == null) { + die("Could not find upstream repository, please specify one with --from"); + } var upstreamPullPath = remotes.contains(upstream) ? Remote.toURI(repo.pullPath(upstream)) : URI.create(upstream); @@ -165,14 +175,12 @@ public static void main(String[] args) throws IOException, InterruptedException System.out.println("done"); } - if (arguments.contains("fetch")) { - int err = fetch(); - if (err != 0) { - System.exit(err); - } + var shouldPull = arguments.contains("pull"); + if (!shouldPull) { + var lines = repo.config("sync.pull"); + shouldPull = lines.size() == 1 && lines.get(0).toLowerCase().equals("always"); } - - if (arguments.contains("pull")) { + if (shouldPull) { int err = pull(); if (err != 0) { System.exit(err); diff --git a/cli/src/main/java/org/openjdk/skara/cli/GitWebrev.java b/cli/src/main/java/org/openjdk/skara/cli/GitWebrev.java index a21a85eb2..a7daf9f00 100644 --- a/cli/src/main/java/org/openjdk/skara/cli/GitWebrev.java +++ b/cli/src/main/java/org/openjdk/skara/cli/GitWebrev.java @@ -36,6 +36,7 @@ import java.nio.file.*; import java.util.*; import java.util.regex.Pattern; +import java.util.stream.Collectors; public class GitWebrev { private static void clearDirectory(Path directory) { @@ -134,7 +135,13 @@ private static void generate(String[] args) throws IOException { .helptext("Print the version of this tool") .optional()); - var parser = new ArgumentParser("git webrev", flags); + var inputs = List.of( + Input.position(0) + .describe("FILE") + .singular() + .optional()); + + var parser = new ArgumentParser("git webrev", flags, inputs); var arguments = parser.parse(args); var version = Version.fromManifest().orElse("unknown"); @@ -156,15 +163,17 @@ private static void generate(String[] args) throws IOException { if (upstream == null) { try { var remote = isMercurial ? "default" : "origin"; - var pullPath = repo.pullPath(remote); - var uri = new URI(pullPath); - var host = uri.getHost(); - var path = uri.getPath(); - if (host != null && path != null) { - if (host.equals("github.com") && path.startsWith("/openjdk/")) { - upstream = "https://github.com" + path; - } else if (host.equals("openjdk.java.net")) { - upstream = "https://openjdk.java.net" + path; + if (repo.remotes().contains(remote)) { + var pullPath = repo.pullPath(remote); + var uri = new URI(pullPath); + var host = uri.getHost(); + var path = uri.getPath(); + if (host != null && path != null) { + if (host.equals("github.com") && path.startsWith("/openjdk/")) { + upstream = "https://github.com" + path; + } else if (host.equals("openjdk.java.net")) { + upstream = "https://openjdk.java.net" + path; + } } } } catch (URISyntaxException e) { @@ -244,6 +253,11 @@ private static void generate(String[] args) throws IOException { clearDirectory(output); } + List files = List.of(); + if (arguments.at(0).isPresent()) { + var path = arguments.at(0).via(Path::of); + files = Files.readAllLines(path).stream().map(Path::of).collect(Collectors.toList()); + } Webrev.repository(repo) .output(output) .title(title) @@ -251,6 +265,7 @@ private static void generate(String[] args) throws IOException { .username(username) .issue(issue) .version(version) + .files(files) .generate(rev); } diff --git a/cli/src/main/java/org/openjdk/skara/cli/Remote.java b/cli/src/main/java/org/openjdk/skara/cli/Remote.java index 334f597cd..c71f3ca6e 100644 --- a/cli/src/main/java/org/openjdk/skara/cli/Remote.java +++ b/cli/src/main/java/org/openjdk/skara/cli/Remote.java @@ -34,6 +34,9 @@ public static URI toWebURI(String remotePath) throws IOException { if (remotePath.startsWith("git+")) { remotePath = remotePath.substring("git+".length()); } + if (remotePath.endsWith(".git")) { + remotePath = remotePath.substring(0, remotePath.length() - ".git".length()); + } if (remotePath.startsWith("http")) { return URI.create(remotePath); } else { diff --git a/deps.env b/deps.env index 09ca6d999..b7d158e60 100644 --- a/deps.env +++ b/deps.env @@ -1,11 +1,11 @@ -JDK_LINUX_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_linux-x64_bin.tar.gz" -JDK_LINUX_SHA256="b43bc15f4934f6d321170419f2c24451486bc848a2179af5e49d10721438dd56" +JDK_LINUX_X64_URL="https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_linux-x64_bin.tar.gz" +JDK_LINUX_X64_SHA256="2e01716546395694d3fad54c9b36d1cd46c5894c06f72d156772efbcf4b41335" -JDK_MACOS_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_osx-x64_bin.tar.gz" -JDK_MACOS_SHA256="52164a04db4d3fdfe128cfc7b868bc4dae52d969f03d53ae9d4239fe783e1a3a" +JDK_MACOS_X64_URL="https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_osx-x64_bin.tar.gz" +JDK_MACOS_X64_SHA256="593c5c9dc0978db21b06d6219dc8584b76a59c79d57e6ec1b28ad0d848a7713f" -JDK_WINDOWS_URL="https://download.java.net/java/GA/jdk12/GPL/openjdk-12_windows-x64_bin.zip" -JDK_WINDOWS_SHA256="35a8d018f420fb05fe7c2aa9933122896ca50bd23dbd373e90d8e2f3897c4e92" +JDK_WINDOWS_X64_URL="https://download.java.net/java/GA/jdk13.0.1/cec27d702aa74d5a8630c65ae61e4305/9/GPL/openjdk-13.0.1_windows-x64_bin.zip" +JDK_WINDOWS_X64_SHA256="438a6920f1851b1eeb6f09f05d9f91c4423c6586f7a1a7ccbb19df76ea5901ee" -GRADLE_URL="https://services.gradle.org/distributions/gradle-5.6.2-bin.zip" -GRADLE_SHA256="32fce6628848f799b0ad3205ae8db67d0d828c10ffe62b748a7c0d9f4a5d9ee0" +GRADLE_URL="https://services.gradle.org/distributions/gradle-6.0-bin.zip" +GRADLE_SHA256="5a3578b9f0bb162f5e08cf119f447dfb8fa950cedebb4d2a977e912a11a74b91" diff --git a/email/src/main/java/org/openjdk/skara/email/Email.java b/email/src/main/java/org/openjdk/skara/email/Email.java index 8d06b596f..aafe09318 100644 --- a/email/src/main/java/org/openjdk/skara/email/Email.java +++ b/email/src/main/java/org/openjdk/skara/email/Email.java @@ -146,6 +146,14 @@ public static EmailBuilder reply(Email parent, String subject, String body) { .header("References", references); } + public static EmailBuilder reparent(Email newParent, Email email) { + var currentParent = email.headerValue("In-Reply-To"); + var currentRefs = email.headerValue("References"); + + return from(email).header("In-Reply-To", newParent.id.toString()) + .header("References", currentRefs.replace(currentParent, newParent.id.toString())); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/email/src/main/java/org/openjdk/skara/email/MimeText.java b/email/src/main/java/org/openjdk/skara/email/MimeText.java index 10dcda8e0..323fd69d3 100644 --- a/email/src/main/java/org/openjdk/skara/email/MimeText.java +++ b/email/src/main/java/org/openjdk/skara/email/MimeText.java @@ -24,7 +24,7 @@ import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; -import java.util.Base64; +import java.util.*; import java.util.regex.Pattern; public class MimeText { @@ -33,31 +33,61 @@ public class MimeText { private final static Pattern decodeQuotedPrintablePattern = Pattern.compile("=([0-9A-F]{2})"); public static String encode(String raw) { - var quoteMatcher = encodePattern.matcher(raw); - return quoteMatcher.replaceAll(mo -> "=?utf-8?b?" + Base64.getEncoder().encodeToString(String.valueOf(mo.group(1)).getBytes(StandardCharsets.UTF_8)) + "?="); + var words = raw.split(" "); + var encodedWords = new ArrayList(); + var lastEncoded = false; + for (var word : words) { + var needsQuotePattern = encodePattern.matcher(word); + if (needsQuotePattern.find()) { + if (lastEncoded) { + // Spaces between encoded words are ignored, so add an explicit one + encodedWords.add("=?UTF-8?B?IA==?="); + } + encodedWords.add("=?UTF-8?B?" + Base64.getEncoder().encodeToString(word.getBytes(StandardCharsets.UTF_8)) + "?="); + lastEncoded = true; + } else { + encodedWords.add(word); + lastEncoded = false; + } + } + return String.join(" ", encodedWords); } public static String decode(String encoded) { + var decoded = new StringBuilder(); var quotedMatcher = decodePattern.matcher(encoded); - return quotedMatcher.replaceAll(mo -> { - try { - if (mo.group(2).toUpperCase().equals("B")) { - return new String(Base64.getDecoder().decode(mo.group(3)), mo.group(1)); - } else { - var quotedPrintableMatcher = decodeQuotedPrintablePattern.matcher(mo.group(3)); - return quotedPrintableMatcher.replaceAll(qmo -> { - var byteValue = new byte[1]; - byteValue[0] = (byte)Integer.parseInt(qmo.group(1), 16); - try { - return new String(byteValue, mo.group(1)); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - }); + var lastMatchEnd = 0; + while (quotedMatcher.find()) { + if (quotedMatcher.start() > lastMatchEnd) { + var separator = encoded.substring(lastMatchEnd, quotedMatcher.start()); + if (!separator.isBlank()) { + decoded.append(separator); + } + } + if (quotedMatcher.group(2).toUpperCase().equals("B")) { + try { + decoded.append(new String(Base64.getDecoder().decode(quotedMatcher.group(3)), quotedMatcher.group(1))); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); } - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); + } else { + var quotedDecodedSpaces = quotedMatcher.group(3).replace("_", " "); + var quotedPrintableMatcher = decodeQuotedPrintablePattern.matcher(quotedDecodedSpaces); + decoded.append(quotedPrintableMatcher.replaceAll(qmo -> { + var byteValue = new byte[1]; + byteValue[0] = (byte)Integer.parseInt(qmo.group(1), 16); + try { + return new String(byteValue, quotedMatcher.group(1)); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + })); } - }); + lastMatchEnd = quotedMatcher.end(); + } + if (lastMatchEnd < encoded.length()) { + decoded.append(encoded, lastMatchEnd, encoded.length()); + } + return decoded.toString(); } } diff --git a/email/src/main/java/org/openjdk/skara/email/SMTP.java b/email/src/main/java/org/openjdk/skara/email/SMTP.java index 0884db4a0..28b655ac6 100644 --- a/email/src/main/java/org/openjdk/skara/email/SMTP.java +++ b/email/src/main/java/org/openjdk/skara/email/SMTP.java @@ -28,18 +28,19 @@ import java.time.Duration; import java.time.format.DateTimeFormatter; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Limited SMTP client implementation - only compatibility requirement (currently) is the OpenJDK * mailing list servers. */ public class SMTP { - private static Pattern initReply = Pattern.compile("220 .*"); + private static Pattern initReply = Pattern.compile("^220 .*"); private static Pattern ehloReply = Pattern.compile("^250 .*"); private static Pattern mailReply = Pattern.compile("^250 .*"); private static Pattern rcptReply = Pattern.compile("^250 .*"); - private static Pattern dataReply = Pattern.compile("354 Enter.*"); - private static Pattern doneReply = Pattern.compile("250 .*"); + private static Pattern dataReply = Pattern.compile("^354 .*"); + private static Pattern doneReply = Pattern.compile("^250 .*"); public static void send(String server, EmailAddress recipient, Email email) throws IOException { send(server, recipient, email, Duration.ofMinutes(30)); @@ -74,7 +75,10 @@ public static void send(String server, EmailAddress recipient, Email email, Dura session.sendCommand("Subject: " + MimeText.encode(email.subject())); session.sendCommand("Content-type: text/plain; charset=utf-8"); session.sendCommand(""); - session.sendCommand(email.body()); + var escapedBody = email.body().lines() + .map(line -> line.startsWith(".") ? "." + line : line) + .collect(Collectors.joining("\n")); + session.sendCommand(escapedBody); session.sendCommand(".", doneReply); session.sendCommand("QUIT"); } diff --git a/email/src/test/java/org/openjdk/skara/email/EmailTests.java b/email/src/test/java/org/openjdk/skara/email/EmailTests.java index 1bc0ac4f3..9fb38726e 100644 --- a/email/src/test/java/org/openjdk/skara/email/EmailTests.java +++ b/email/src/test/java/org/openjdk/skara/email/EmailTests.java @@ -67,6 +67,24 @@ void buildFrom() { assertEquals(original, copy); } + @Test + void reparent() { + var first = Email.create(EmailAddress.from("A", "a@b.c"), "First", "body") + .recipient(EmailAddress.from("B", "b@b.c")) + .build(); + var second = Email.create(EmailAddress.from("A", "a@b.c"), "Second", "body") + .recipient(EmailAddress.from("B", "b@b.c")) + .build(); + var reply = Email.reply(first, "The reply", "reply body") + .author(EmailAddress.from("C", "c@b.c")) + .build(); + assertEquals(first.id().toString(), reply.headerValue("In-Reply-To")); + assertEquals(first.id().toString(), reply.headerValue("References")); + var updated = Email.reparent(second, reply).build(); + assertEquals(second.id().toString(), updated.headerValue("In-Reply-To")); + assertEquals(second.id().toString(), updated.headerValue("References")); + } + @Test void caseInsensitiveHeaders() { var mail = Email.parse("Message-ID: \n" + diff --git a/email/src/test/java/org/openjdk/skara/email/MimeTextTests.java b/email/src/test/java/org/openjdk/skara/email/MimeTextTests.java index efb5e0e56..ca6b82635 100644 --- a/email/src/test/java/org/openjdk/skara/email/MimeTextTests.java +++ b/email/src/test/java/org/openjdk/skara/email/MimeTextTests.java @@ -28,17 +28,59 @@ class MimeTextTests { @Test - void encode() { - assertEquals("=?utf-8?b?w6XDpMO2?=", MimeText.encode("åäö")); + void simple() { + var encoded = "=?UTF-8?B?w6XDpMO2?="; + var decoded = "åäö"; + assertEquals(encoded, MimeText.encode(decoded)); + assertEquals(decoded, MimeText.decode(encoded)); } @Test - void decode() { - assertEquals("åäö", MimeText.decode("=?utf-8?b?w6XDpMO2?=")); + void mixed() { + var encoded = "=?UTF-8?B?VMOpc3Q=?="; + var decoded = "Tést"; + assertEquals(encoded, MimeText.encode(decoded)); + assertEquals(decoded, MimeText.decode(encoded)); + } + + @Test + void multipleWords() { + var encoded = "This is a =?UTF-8?B?dMOpc3Q=?= of =?UTF-8?B?bcO8bHRpcGxl?= words"; + var decoded = "This is a tést of mültiple words"; + assertEquals(encoded, MimeText.encode(decoded)); + assertEquals(decoded, MimeText.decode(encoded)); + } + + @Test + void concatenateTokens() { + var encoded = "=?UTF-8?B?VMOpc3Q=?= =?UTF-8?B?IA==?= =?UTF-8?B?VMOpc3Q=?="; + var decoded = "Tést Tést"; + assertEquals(encoded, MimeText.encode(decoded)); + assertEquals(decoded, MimeText.decode(encoded)); + } + + @Test + void preserveSpaces() { + var encoded = "spac es"; + var decoded = "spac es"; + assertEquals(encoded, MimeText.encode(decoded)); + assertEquals(decoded, MimeText.decode(encoded)); + } + + @Test + void decodeSpaces() { + var encoded = "=?UTF-8?B?VMOpc3Q=?= =?UTF-8?B?VMOpc3Q=?= and "; + var decoded = "TéstTést and "; + assertEquals(decoded, MimeText.decode(encoded)); } @Test void decodeIsoQ() { assertEquals("Bä", MimeText.decode("=?iso-8859-1?Q?B=E4?=")); } + + @Test + void decodeIsoQSpaces() { + assertEquals("Bä Bä Bä", MimeText.decode("=?iso-8859-1?Q?B=E4_B=E4=20B=E4?=")); + } } diff --git a/email/src/test/java/org/openjdk/skara/email/SMTPTests.java b/email/src/test/java/org/openjdk/skara/email/SMTPTests.java index 3fc5ea7c4..0cb7ab506 100644 --- a/email/src/test/java/org/openjdk/skara/email/SMTPTests.java +++ b/email/src/test/java/org/openjdk/skara/email/SMTPTests.java @@ -22,24 +22,19 @@ */ package org.openjdk.skara.email; -import org.junit.jupiter.api.condition.DisabledOnOs; -import org.junit.jupiter.api.condition.OS; import org.openjdk.skara.test.SMTPServer; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.*; import java.io.IOException; import java.time.Duration; -import java.util.logging.Logger; import static org.junit.jupiter.api.Assertions.*; class SMTPTests { - private final static Logger log = Logger.getLogger("org.openjdk.skara.email");; - @Test void simple() throws IOException { - log.info("Hello"); try (var server = new SMTPServer()) { var sender = EmailAddress.from("Test", "test@test.email"); var recipient = EmailAddress.from("Dest", "dest@dest.email"); @@ -53,7 +48,6 @@ void simple() throws IOException { @Test void withHeader() throws IOException { - log.info("Hello"); try (var server = new SMTPServer()) { var sender = EmailAddress.from("Test", "test@test.email"); var author = EmailAddress.from("Auth", "auth@test.email"); @@ -73,7 +67,6 @@ void withHeader() throws IOException { @Test @DisabledOnOs(OS.WINDOWS) void encoded() throws IOException { - log.info("Hello"); try (var server = new SMTPServer()) { var sender = EmailAddress.from("Señor Dévèlöper", "test@test.email"); var recipient = EmailAddress.from("Dêst", "dest@dest.email"); @@ -90,7 +83,6 @@ void encoded() throws IOException { @Test void timeout() throws IOException { - log.info("Hello"); try (var server = new SMTPServer()) { var sender = EmailAddress.from("Test", "test@test.email"); var recipient = EmailAddress.from("Dest", "dest@dest.email"); @@ -99,4 +91,17 @@ void timeout() throws IOException { assertThrows(RuntimeException.class, () -> SMTP.send(server.address(), recipient, sentMail, Duration.ZERO)); } } + + @Test + void withDot() throws IOException { + try (var server = new SMTPServer()) { + var sender = EmailAddress.from("Test", "test@test.email"); + var recipient = EmailAddress.from("Dest", "dest@dest.email"); + var sentMail = Email.create(sender, "Subject", "Body\n.\nMore text").recipient(recipient).build(); + + SMTP.send(server.address(), recipient, sentMail); + var email = server.receive(Duration.ofSeconds(10)); + assertEquals(sentMail, email); + } + } } diff --git a/encoding/src/main/java/org/openjdk/skara/base85/Base85.java b/encoding/src/main/java/org/openjdk/skara/encoding/Base85.java similarity index 100% rename from encoding/src/main/java/org/openjdk/skara/base85/Base85.java rename to encoding/src/main/java/org/openjdk/skara/encoding/Base85.java diff --git a/forge/build.gradle b/forge/build.gradle index 30d028cef..ba1a42bfb 100644 --- a/forge/build.gradle +++ b/forge/build.gradle @@ -28,6 +28,8 @@ module { requires 'org.junit.jupiter.api' requires 'jdk.httpserver' opens 'org.openjdk.skara.forge' to 'org.junit.platform.commons' + opens 'org.openjdk.skara.forge.github' to 'org.junit.platform.commons' + opens 'org.openjdk.skara.forge.gitlab' to 'org.junit.platform.commons' } } diff --git a/forge/src/main/java/module-info.java b/forge/src/main/java/module-info.java index 243ff71e4..824f6d41c 100644 --- a/forge/src/main/java/module-info.java +++ b/forge/src/main/java/module-info.java @@ -34,4 +34,8 @@ requires java.logging; exports org.openjdk.skara.forge; + + uses org.openjdk.skara.forge.ForgeFactory; + + provides org.openjdk.skara.forge.ForgeFactory with org.openjdk.skara.forge.github.GitHubForgeFactory, org.openjdk.skara.forge.gitlab.GitLabForgeFactory; } diff --git a/forge/src/main/java/org/openjdk/skara/forge/Forge.java b/forge/src/main/java/org/openjdk/skara/forge/Forge.java index fff5464d3..6f7b9a514 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/Forge.java +++ b/forge/src/main/java/org/openjdk/skara/forge/Forge.java @@ -23,17 +23,44 @@ package org.openjdk.skara.forge; import org.openjdk.skara.host.*; +import org.openjdk.skara.json.JSONObject; import java.net.URI; +import java.util.*; +import java.util.stream.Collectors; public interface Forge extends Host { - HostedRepository repository(String name); + Optional repository(String name); boolean supportsReviewBody(); - static Forge from(URI uri, PersonalAccessToken pat) { - return ForgeFactory.createFromURI(uri, pat); + static Forge from(String name, URI uri, Credential credential, JSONObject configuration) { + var factory = ForgeFactory.getForgeFactories().stream() + .filter(f -> f.name().equals(name)) + .findFirst(); + if (factory.isEmpty()) { + throw new RuntimeException("No forge factory named '" + name + "' found - check module path"); + } + return factory.get().create(uri, credential, configuration); } - static Forge from(URI uri) { - return ForgeFactory.createFromURI(uri, null); + + static Optional from(URI uri, Credential credential, JSONObject configuration) { + var factories = ForgeFactory.getForgeFactories().stream() + .sorted(Comparator.comparing(f -> !uri.getHost().contains(f.name()))) + .collect(Collectors.toList()); + for (var factory : factories) { + var forge = factory.create(uri, credential, configuration); + if (forge.isValid()) { + return Optional.of(forge); + } + } + return Optional.empty(); + } + + static Optional from(URI uri, Credential credential) { + return from(uri, credential, null); + } + + static Optional from(URI uri) { + return from(uri, null); } } diff --git a/forge/src/main/java/org/openjdk/skara/forge/ForgeFactory.java b/forge/src/main/java/org/openjdk/skara/forge/ForgeFactory.java index eae341827..6af275428 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/ForgeFactory.java +++ b/forge/src/main/java/org/openjdk/skara/forge/ForgeFactory.java @@ -22,56 +22,28 @@ */ package org.openjdk.skara.forge; -import org.openjdk.skara.host.*; +import org.openjdk.skara.host.Credential; +import org.openjdk.skara.json.JSONObject; import java.net.URI; -import java.util.regex.Pattern; - -public class ForgeFactory { - public static Forge createGitHubHost(URI uri, Pattern webUriPattern, String webUriReplacement, String keyFile, String issue, String id) { - var app = new GitHubApplication(keyFile, issue, id); - return new GitHubHost(uri, app, webUriPattern, webUriReplacement); - } - - public static Forge createGitHubHost(URI uri, PersonalAccessToken pat) { - if (pat != null) { - return new GitHubHost(uri, pat); - } else { - return new GitHubHost(uri); - } - } - - public static Forge createGitLabHost(URI uri, PersonalAccessToken pat) { - if (pat != null) { - return new GitLabHost(uri, pat); - } else { - return new GitLabHost(uri); - } - } - - public static Forge createFromURI(URI uri, PersonalAccessToken pat) throws IllegalArgumentException { - // Short-circuit - if (uri.toString().contains("github")) { - return createGitHubHost(uri, pat); - } else if (uri.toString().contains("gitlab")) { - return createGitLabHost(uri, pat); - } - - try { - var gitLabHost = createGitLabHost(uri, pat); - if (gitLabHost.isValid()) { - return gitLabHost; - } - } catch (RuntimeException e) { - try { - var gitHubHost = createGitHubHost(uri, pat); - if (gitHubHost.isValid()) { - return gitHubHost; - } - } catch (RuntimeException ignored) { - } - } - - throw new IllegalArgumentException("Unable to detect host type from URI: " + uri); +import java.util.*; +import java.util.stream.*; + +public interface ForgeFactory { + /** + * A user-friendly name for the given forge, used for configuration section naming. Should be lower case. + * @return + */ + String name(); + + /** + * Instantiate an instance of this forge. + * @return + */ + Forge create(URI uri, Credential credential, JSONObject configuration); + + static List getForgeFactories() { + return StreamSupport.stream(ServiceLoader.load(ForgeFactory.class).spliterator(), false) + .collect(Collectors.toList()); } } diff --git a/forge/src/main/java/org/openjdk/skara/forge/PositionMapper.java b/forge/src/main/java/org/openjdk/skara/forge/PositionMapper.java deleted file mode 100644 index 8f6feeacf..000000000 --- a/forge/src/main/java/org/openjdk/skara/forge/PositionMapper.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package org.openjdk.skara.forge; - -import java.util.*; -import java.util.logging.Logger; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -class PositionMapper { - private static final Pattern filePattern = Pattern.compile("^diff --git a/(.*) b/.*$"); - private static final Pattern hunkPattern = Pattern.compile("^@@ -(\\d+)(?:,\\d+)? \\+(\\d+)(?:,\\d+)? @@.*"); - - private static class PositionOffset { - int position; - int line; - } - - private final Map> fileDiffs = new HashMap<>(); - private final Logger log = Logger.getLogger("org.openjdk.skara.host.github"); - - private PositionMapper(List lines) { - int position = 0; - var latestList = new ArrayList(); - - for (var line : lines) { - var fileMatcher = filePattern.matcher(line); - if (fileMatcher.matches()) { - latestList = new ArrayList<>(); - fileDiffs.put(fileMatcher.group(1), latestList); - continue; - } - var hunkMatcher = hunkPattern.matcher(line); - if (hunkMatcher.matches()) { - var positionOffset = new PositionOffset(); - if (latestList.isEmpty()) { - position = 1; - positionOffset.position = 1; - } else { - positionOffset.position = position + 1; - } - positionOffset.line = Integer.parseInt(hunkMatcher.group(2)); - latestList.add(positionOffset); - } - position++; - } - } - - int positionToLine(String file, int position) { - if (!fileDiffs.containsKey(file)) { - throw new IllegalArgumentException("Unknown file " + file); - } - var positionOffsets = fileDiffs.get(file); - PositionOffset activeOffset = null; - for (var offset : positionOffsets) { - if (offset.position > position) { - break; - } - activeOffset = offset; - } - if (activeOffset == null) { - log.warning("No matching line found (position: " + position + " file: " + file + ")"); - return -1; - } - return activeOffset.line + (position - activeOffset.position); - } - - int lineToPosition(String file, int line) { - if (!fileDiffs.containsKey(file)) { - throw new IllegalArgumentException("Unknown file " + file); - } - var positionOffsets = fileDiffs.get(file); - PositionOffset activeOffset = null; - for (var offset : positionOffsets) { - if (offset.line > line) { - break; - } - activeOffset = offset; - } - if (activeOffset == null) { - log.warning("No matching position found (line: " + line + " file: " + file + ")"); - return -1; - } - return activeOffset.position + (line - activeOffset.line); - } - - static PositionMapper parse(String diff) { - return new PositionMapper(diff.lines().collect(Collectors.toList())); - } -} diff --git a/forge/src/main/java/org/openjdk/skara/forge/PullRequestUpdateCache.java b/forge/src/main/java/org/openjdk/skara/forge/PullRequestUpdateCache.java index c67c4947b..f9af93680 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/PullRequestUpdateCache.java +++ b/forge/src/main/java/org/openjdk/skara/forge/PullRequestUpdateCache.java @@ -22,6 +22,8 @@ */ package org.openjdk.skara.forge; +import org.openjdk.skara.forge.gitlab.GitLabMergeRequest; + import java.time.ZonedDateTime; import java.util.*; import java.util.logging.Logger; diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitHubApplication.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubApplication.java similarity index 95% rename from forge/src/main/java/org/openjdk/skara/forge/GitHubApplication.java rename to forge/src/main/java/org/openjdk/skara/forge/github/GitHubApplication.java index bb8955ef9..0d6a09e7f 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitHubApplication.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubApplication.java @@ -20,16 +20,15 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.github; -import org.openjdk.skara.network.URIBuilder; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.URIBuilder; import java.io.*; import java.net.URI; import java.net.http.*; import java.nio.charset.StandardCharsets; -import java.nio.file.*; import java.security.*; import java.security.spec.*; import java.time.*; @@ -104,7 +103,7 @@ public GitHubConfigurationError(String message) { } } - public GitHubApplication(String keyFile, String issue, String id) { + public GitHubApplication(String key, String issue, String id) { log = Logger.getLogger("org.openjdk.host.github"); @@ -112,14 +111,13 @@ public GitHubApplication(String keyFile, String issue, String id) { this.issue = issue; this.id = id; - key = loadPkcs8PemFromFile(keyFile); + this.key = loadPkcs8PemFromString(key); jwt = new Token(this::generateJsonWebToken, Duration.ofMinutes(5)); installationToken = new Token(this::generateInstallationToken, Duration.ofMinutes(30)); } - private PrivateKey loadPkcs8PemFromFile(String keyFile) { + private PrivateKey loadPkcs8PemFromString(String pem) { try { - var pem = new String(Files.readAllBytes(Paths.get(keyFile))); var pemPattern = Pattern.compile("^-*BEGIN PRIVATE KEY-*$(.*)^-*END PRIVATE KEY-*", Pattern.DOTALL | Pattern.MULTILINE); var keyString = pemPattern.matcher(pem).replaceFirst("$1"); @@ -127,8 +125,8 @@ private PrivateKey loadPkcs8PemFromFile(String keyFile) { var rawKey = Base64.getMimeDecoder().decode(keyString); var factory = KeyFactory.getInstance("RSA"); return factory.generatePrivate(new PKCS8EncodedKeySpec(rawKey)); - } catch (NoSuchAlgorithmException | InvalidKeySpecException | IOException e) { - throw new GitHubConfigurationError("Unable to load private key (" + keyFile + ": " + e + ")"); + } catch (NoSuchAlgorithmException | InvalidKeySpecException e) { + throw new GitHubConfigurationError("Unable to load private key (" + e + ")"); } } diff --git a/forge/src/main/java/org/openjdk/skara/forge/github/GitHubForgeFactory.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubForgeFactory.java new file mode 100644 index 000000000..42bb59027 --- /dev/null +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubForgeFactory.java @@ -0,0 +1,39 @@ +package org.openjdk.skara.forge.github; + +import org.openjdk.skara.forge.*; +import org.openjdk.skara.host.Credential; +import org.openjdk.skara.json.JSONObject; + +import java.net.URI; +import java.util.regex.Pattern; + +public class GitHubForgeFactory implements ForgeFactory { + @Override + public String name() { + return "github"; + } + + @Override + public Forge create(URI uri, Credential credential, JSONObject configuration) { + Pattern webUriPattern = null; + String webUriReplacement = null; + if (configuration != null && configuration.contains("weburl")) { + webUriPattern = Pattern.compile(configuration.get("weburl").get("pattern").asString()); + webUriReplacement = configuration.get("weburl").get("replacement").asString(); + } + + if (credential != null) { + if (credential.username().contains(";")) { + var separator = credential.username().indexOf(";"); + var id = credential.username().substring(0, separator); + var installation = credential.username().substring(separator + 1); + var app = new GitHubApplication(credential.password(), id, installation); + return new GitHubHost(uri, app, webUriPattern, webUriReplacement); + } else { + return new GitHubHost(uri, credential, webUriPattern, webUriReplacement); + } + } else { + return new GitHubHost(uri, webUriPattern, webUriReplacement); + } + } +} diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitHubHost.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java similarity index 73% rename from forge/src/main/java/org/openjdk/skara/forge/GitHubHost.java rename to forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java index 3414194c9..d5ba49572 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitHubHost.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubHost.java @@ -20,15 +20,18 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.github; +import org.openjdk.skara.forge.*; import org.openjdk.skara.host.*; -import org.openjdk.skara.network.*; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; +import java.io.IOException; import java.net.*; import java.nio.charset.StandardCharsets; -import java.util.Arrays; +import java.util.*; +import java.util.logging.Logger; import java.util.regex.Pattern; public class GitHubHost implements Forge { @@ -36,9 +39,10 @@ public class GitHubHost implements Forge { private final Pattern webUriPattern; private final String webUriReplacement; private final GitHubApplication application; - private final PersonalAccessToken pat; + private final Credential pat; private final RestRequest request; private HostUser currentUser; + private final Logger log = Logger.getLogger("org.openjdk.skara.forge.github"); public GitHubHost(URI uri, GitHubApplication application, Pattern webUriPattern, String webUriReplacement) { this.uri = uri; @@ -53,15 +57,15 @@ public GitHubHost(URI uri, GitHubApplication application, Pattern webUriPattern, .build(); request = new RestRequest(baseApi, () -> Arrays.asList( - "Authorization", "token " + getInstallationToken(), + "Authorization", "token " + getInstallationToken().orElseThrow(), "Accept", "application/vnd.github.machine-man-preview+json", "Accept", "application/vnd.github.antiope-preview+json")); } - public GitHubHost(URI uri, PersonalAccessToken pat) { + public GitHubHost(URI uri, Credential pat, Pattern webUriPattern, String webUriReplacement) { this.uri = uri; - this.webUriPattern = null; - this.webUriReplacement = null; + this.webUriPattern = webUriPattern; + this.webUriReplacement = webUriReplacement; this.pat = pat; this.application = null; @@ -71,13 +75,13 @@ public GitHubHost(URI uri, PersonalAccessToken pat) { .build(); request = new RestRequest(baseApi, () -> Arrays.asList( - "Authorization", "token " + pat.token())); + "Authorization", "token " + getInstallationToken().orElseThrow())); } - public GitHubHost(URI uri) { + GitHubHost(URI uri, Pattern webUriPattern, String webUriReplacement) { this.uri = uri; - this.webUriPattern = null; - this.webUriReplacement = null; + this.webUriPattern = webUriPattern; + this.webUriReplacement = webUriReplacement; this.pat = null; this.application = null; @@ -110,12 +114,16 @@ URI getWebURI(String endpoint) { return URIBuilder.base(matcher.replaceAll(webUriReplacement)).build(); } - String getInstallationToken() { + Optional getInstallationToken() { if (application != null) { - return application.getInstallationToken(); - } else { - return pat.token(); + return Optional.of(application.getInstallationToken()); + } + + if (pat != null) { + return Optional.of(pat.password()); } + + return Optional.empty(); } private String getFullName(String userName) { @@ -135,10 +143,20 @@ HostUser parseUserObject(JSONValue json) { @Override public boolean isValid() { - var endpoints = request.get("") - .onError(response -> JSON.of()) - .execute(); - return !endpoints.isNull(); + try { + var endpoints = request.get("") + .executeUnparsed(); + var parsed = JSON.parse(endpoints); + if (parsed != null && parsed.contains("current_user_url")) { + return true; + } else { + log.fine("Error during GitHub host validation: unexpected endpoint list: " + endpoints); + return false; + } + } catch (IOException e) { + log.fine("Error during GitHub host validation: " + e); + return false; + } } JSONObject getProjectInfo(String name) { @@ -155,14 +173,21 @@ JSONObject runSearch(String query) { } @Override - public HostedRepository repository(String name) { - return new GitHubRepository(this, name); + public Optional repository(String name) { + try { + return Optional.of(new GitHubRepository(this, name)); + } catch (Throwable t) { + return Optional.empty(); + } } @Override public HostUser user(String username) { var details = request.get("users/" + URLEncoder.encode(username, StandardCharsets.UTF_8)).execute().asObject(); + return asHostUser(details); + } + private static HostUser asHostUser(JSONObject details) { // Always present var login = details.get("login").asString(); var id = details.get("id").asInt(); @@ -182,7 +207,11 @@ public HostUser currentUser() { var appName = appDetails.get("name").asString() + "[bot]"; currentUser = user(appName); } else if (pat != null) { - currentUser = user(pat.userName()); + // Cannot always trust username in PAT, e.g. Git Credential Manager + // on Windows always return "PersonalAccessToken" as username. + // Query GitHub for the username instead. + var details = request.get("user").execute().asObject(); + currentUser = asHostUser(details); } else { throw new IllegalStateException("No credentials present"); } diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitHubPullRequest.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java similarity index 88% rename from forge/src/main/java/org/openjdk/skara/forge/GitHubPullRequest.java rename to forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java index fc3a6644f..b795e46e2 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitHubPullRequest.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubPullRequest.java @@ -20,12 +20,13 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.github; -import org.openjdk.skara.host.*; +import org.openjdk.skara.forge.*; +import org.openjdk.skara.host.HostUser; import org.openjdk.skara.issuetracker.*; -import org.openjdk.skara.network.*; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; import org.openjdk.skara.vcs.Hash; import java.net.URI; @@ -117,14 +118,35 @@ public void addReview(Review.Verdict verdict, String body) { .execute(); } - private ReviewComment parseReviewComment(ReviewComment parent, JSONObject json, PositionMapper diff) { + private ReviewComment parseReviewComment(ReviewComment parent, JSONObject json) { var author = host.parseUserField(json); var threadId = parent == null ? json.get("id").toString() : parent.threadId(); + + int line = json.get("original_line").asInt(); + var hash = new Hash(json.get("original_commit_id").asString()); + var path = json.get("path").asString(); + + if (json.get("side").asString().equals("LEFT")) { + var commitInfo = request.get("commits/" + hash).execute(); + + // It's possible that the file in question was renamed / deleted in an earlier commit, would + // need to parse all the commits in the PR to be sure. But this should cover most cases. + hash = new Hash(commitInfo.get("parents").asArray().get(0).get("sha").asString()); + for (var file : commitInfo.get("files").asArray()) { + if (file.get("filename").asString().equals(path)) { + if (file.get("status").asString().equals("renamed")) { + path = file.get("previous_filename").asString(); + } + break; + } + } + } + var comment = new ReviewComment(parent, threadId, - new Hash(json.get("commit_id").asString()), - json.get("path").asString(), - diff.positionToLine(json.get("path").asString(), json.get("original_position").asInt()), + hash, + path, + line, json.get("id").toString(), json.get("body").asString(), author, @@ -135,45 +157,31 @@ private ReviewComment parseReviewComment(ReviewComment parent, JSONObject json, @Override public ReviewComment addReviewComment(Hash base, Hash hash, String path, int line, String body) { - var rawDiff = request.get("pulls/" + json.get("number").toString()) - .header("Accept", "application/vnd.github.v3.diff") - .executeUnparsed(); - var diff = PositionMapper.parse(rawDiff); - var query = JSON.object() - .put("body", body) - .put("commit_id", hash.hex()) - .put("path", path) - .put("position", diff.lineToPosition(path, line)); + .put("body", body) + .put("commit_id", hash.hex()) + .put("path", path) + .put("side", "RIGHT") + .put("line", line); var response = request.post("pulls/" + json.get("number").toString() + "/comments") - .body(query) - .execute(); - return parseReviewComment(null, response.asObject(), diff); + .body(query) + .execute(); + return parseReviewComment(null, response.asObject()); } @Override public ReviewComment addReviewCommentReply(ReviewComment parent, String body) { - var rawDiff = request.get("pulls/" + json.get("number").toString()) - .header("Accept", "application/vnd.github.v3.diff") - .executeUnparsed(); - var diff = PositionMapper.parse(rawDiff); - var query = JSON.object() .put("body", body) - .put("in_reply_to", Integer.parseInt(parent.threadId())); + .put("in_reply_to", Integer.parseInt(parent.threadId())); var response = request.post("pulls/" + json.get("number").toString() + "/comments") - .body(query) - .execute(); - return parseReviewComment(parent, response.asObject(), diff); + .body(query) + .execute(); + return parseReviewComment(parent, response.asObject()); } @Override public List reviewComments() { - var rawDiff = request.get("pulls/" + json.get("number").toString()) - .header("Accept", "application/vnd.github.v3.diff") - .executeUnparsed(); - var diff = PositionMapper.parse(rawDiff); - var ret = new ArrayList(); var reviewComments = request.get("pulls/" + json.get("number").toString() + "/comments").execute().stream() .map(JSONValue::asObject) @@ -185,7 +193,7 @@ public List reviewComments() { if (reviewComment.contains("in_reply_to_id")) { parent = idToComment.get(reviewComment.get("in_reply_to_id").toString()); } - var comment = parseReviewComment(parent, reviewComment, diff); + var comment = parseReviewComment(parent, reviewComment); idToComment.put(comment.id(), comment); ret.add(comment); } @@ -220,7 +228,9 @@ public String title() { @Override public void setTitle(String title) { - throw new RuntimeException("not implemented yet"); + request.patch("pulls/" + json.get("number").toString()) + .body("title", title) + .execute(); } @Override @@ -303,6 +313,8 @@ public Map checks(Hash hash) { checkBuilder.complete(true, completedAt); break; case "failure": + // fallthrough + case "neutral": checkBuilder.complete(false, completedAt); break; default: @@ -323,12 +335,12 @@ public Map checks(Hash hash) { } return checkBuilder.build(); - })); + }, (a, b) -> b)); } @Override public void createCheck(Check check) { - // update and create are currenly identical operations, both do an HTTP + // update and create are currently identical operations, both do an HTTP // POST to the /repos/:owner/:repo/check-runs endpoint. There is an additional // endpoint explicitly for updating check-runs, but that is not currently used. updateCheck(check); @@ -347,7 +359,7 @@ public void updateCheck(Check check) { outputQuery.put("summary", check.summary().get()); var annotations = JSON.array(); - for (var annotation : check.annotations()) { + for (var annotation : check.annotations().subList(0, Math.min(check.annotations().size(), 50))) { var annotationQuery = JSON.object(); annotationQuery.put("path", annotation.path()); annotationQuery.put("start_line", annotation.startLine()); diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitHubRepository.java b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubRepository.java similarity index 88% rename from forge/src/main/java/org/openjdk/skara/forge/GitHubRepository.java rename to forge/src/main/java/org/openjdk/skara/forge/github/GitHubRepository.java index 1fe31066f..7c05fe037 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitHubRepository.java +++ b/forge/src/main/java/org/openjdk/skara/forge/github/GitHubRepository.java @@ -20,11 +20,11 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.github; -import org.openjdk.skara.issuetracker.Issue; -import org.openjdk.skara.network.*; +import org.openjdk.skara.forge.*; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; import org.openjdk.skara.vcs.*; import java.net.URI; @@ -49,11 +49,19 @@ public class GitHubRepository implements HostedRepository { .appendSubDomain("api") .setPath("/repos/" + repository + "/") .build(); - request = new RestRequest(apiBase, () -> Arrays.asList( - "Authorization", "token " + gitHubHost.getInstallationToken(), + request = new RestRequest(apiBase, () -> { + var headers = new ArrayList<>(List.of( "Accept", "application/vnd.github.machine-man-preview+json", "Accept", "application/vnd.github.antiope-preview+json", - "Accept", "application/vnd.github.shadow-cat-preview+json")); + "Accept", "application/vnd.github.shadow-cat-preview+json", + "Accept", "application/vnd.github.comfort-fade-preview+json")); + var token = gitHubHost.getInstallationToken(); + if (token.isPresent()) { + headers.add("Authorization"); + headers.add("token " + token.get()); + } + return headers; + }); json = gitHubHost.getProjectInfo(repository); var urlPattern = gitHubHost.getWebURI("/" + repository + "/pull/").toString(); pullRequestPattern = Pattern.compile(urlPattern + "(\\d+)"); @@ -142,11 +150,13 @@ public String name() { @Override public URI url() { - return URIBuilder - .base(gitHubHost.getURI()) - .setPath("/" + repository + ".git") - .setAuthentication("x-access-token:" + gitHubHost.getInstallationToken()) - .build(); + var builder = URIBuilder.base(gitHubHost.getURI()) + .setPath("/" + repository + ".git"); + var token = gitHubHost.getInstallationToken(); + if (token.isPresent()) { + builder.setAuthentication("x-access-token:" + token.get()); + } + return builder.build(); } @Override @@ -190,7 +200,7 @@ public Optional parseWebHook(JSONValue body) { @Override public HostedRepository fork() { var response = request.post("forks").execute(); - return gitHubHost.repository(response.get("full_name").asString()); + return gitHubHost.repository(response.get("full_name").asString()).orElseThrow(RuntimeException::new); } @Override diff --git a/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabForgeFactory.java b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabForgeFactory.java new file mode 100644 index 000000000..38fdcedf8 --- /dev/null +++ b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabForgeFactory.java @@ -0,0 +1,23 @@ +package org.openjdk.skara.forge.gitlab; + +import org.openjdk.skara.forge.*; +import org.openjdk.skara.host.Credential; +import org.openjdk.skara.json.JSONObject; + +import java.net.URI; + +public class GitLabForgeFactory implements ForgeFactory { + @Override + public String name() { + return "gitlab"; + } + + @Override + public Forge create(URI uri, Credential credential, JSONObject configuration) { + if (credential != null) { + return new GitLabHost(uri, credential); + } else { + return new GitLabHost(uri); + } + } +} diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitLabHost.java b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabHost.java similarity index 80% rename from forge/src/main/java/org/openjdk/skara/forge/GitLabHost.java rename to forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabHost.java index 9525eea3c..6cb5029a3 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitLabHost.java +++ b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabHost.java @@ -20,32 +20,36 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.gitlab; +import org.openjdk.skara.forge.*; import org.openjdk.skara.host.*; -import org.openjdk.skara.network.*; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; +import java.io.IOException; import java.net.*; import java.nio.charset.StandardCharsets; import java.util.*; +import java.util.logging.Logger; public class GitLabHost implements Forge { private final URI uri; - private final PersonalAccessToken pat; + private final Credential pat; private final RestRequest request; + private final Logger log = Logger.getLogger("org.openjdk.skara.forge.gitlab"); - public GitLabHost(URI uri, PersonalAccessToken pat) { + GitLabHost(URI uri, Credential pat) { this.uri = uri; this.pat = pat; var baseApi = URIBuilder.base(uri) .setPath("/api/v4/") .build(); - request = new RestRequest(baseApi, () -> Arrays.asList("Private-Token", pat.token())); + request = new RestRequest(baseApi, () -> Arrays.asList("Private-Token", pat.password())); } - public GitLabHost(URI uri) { + GitLabHost(URI uri) { this.uri = uri; this.pat = null; @@ -59,16 +63,26 @@ public URI getUri() { return uri; } - public Optional getPat() { + Optional getPat() { return Optional.ofNullable(pat); } @Override public boolean isValid() { - var version = request.get("version") - .onError(r -> JSON.object().put("invalid", true)) - .execute(); - return !version.contains("invalid"); + try { + var version = request.get("version") + .executeUnparsed(); + var parsed = JSON.parse(version); + if (parsed != null && parsed.contains("version")) { + return true; + } else { + log.fine("Error during GitLab host validation: unexpected version: " + version); + return false; + } + } catch (IOException e) { + log.fine("Error during GitLab host validation: " + e); + return false; + } } JSONObject getProjectInfo(String name) { @@ -88,8 +102,12 @@ JSONObject getProjectInfo(String name) { } @Override - public HostedRepository repository(String name) { - return new GitLabRepository(this, name); + public Optional repository(String name) { + try { + return Optional.of(new GitLabRepository(this, name)); + } catch (Throwable t) { + return Optional.empty(); + } } private HostUser parseUserDetails(JSONObject details) { diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitLabMergeRequest.java b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabMergeRequest.java similarity index 96% rename from forge/src/main/java/org/openjdk/skara/forge/GitLabMergeRequest.java rename to forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabMergeRequest.java index a13900f86..b4ad2dc25 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitLabMergeRequest.java +++ b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabMergeRequest.java @@ -20,12 +20,13 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.gitlab; -import org.openjdk.skara.host.*; +import org.openjdk.skara.forge.*; +import org.openjdk.skara.host.HostUser; import org.openjdk.skara.issuetracker.*; -import org.openjdk.skara.network.*; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; import org.openjdk.skara.vcs.Hash; import java.net.URI; @@ -160,11 +161,26 @@ public void addReview(Review.Verdict verdict, String body) { } private ReviewComment parseReviewComment(String discussionId, ReviewComment parent, JSONObject note) { + int line; + String path; + Hash hash; + + // Is the comment on the old or the new version of the file? + if (note.get("position").get("new_line").isNull()) { + line = note.get("position").get("old_line").asInt(); + path = note.get("position").get("old_path").asString(); + hash = new Hash(note.get("position").get("start_sha").asString()); + } else { + line = note.get("position").get("new_line").asInt(); + path = note.get("position").get("new_path").asString(); + hash = new Hash(note.get("position").get("head_sha").asString()); + } + var comment = new ReviewComment(parent, discussionId, - new Hash(note.get("position").get("head_sha").asString()), - note.get("position").get("new_path").asString(), - note.get("position").get("new_line").asInt(), + hash, + path, + line, note.get("id").toString(), note.get("body").asString(), new HostUser(note.get("author").get("id").asInt(), @@ -262,7 +278,9 @@ public String title() { @Override public void setTitle(String title) { - throw new RuntimeException("not implemented yet"); + request.put("") + .body("title", title) + .execute(); } @Override diff --git a/forge/src/main/java/org/openjdk/skara/forge/GitLabRepository.java b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabRepository.java similarity index 97% rename from forge/src/main/java/org/openjdk/skara/forge/GitLabRepository.java rename to forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabRepository.java index fa1c8cf43..faa52ec5a 100644 --- a/forge/src/main/java/org/openjdk/skara/forge/GitLabRepository.java +++ b/forge/src/main/java/org/openjdk/skara/forge/gitlab/GitLabRepository.java @@ -20,8 +20,9 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.gitlab; +import org.openjdk.skara.forge.*; import org.openjdk.skara.json.*; import org.openjdk.skara.network.*; import org.openjdk.skara.vcs.*; @@ -51,7 +52,7 @@ public GitLabRepository(GitLabHost gitLabHost, String projectName) { .build(); request = gitLabHost.getPat() - .map(pat -> new RestRequest(baseApi, () -> Arrays.asList("Private-Token", pat.token()))) + .map(pat -> new RestRequest(baseApi, () -> Arrays.asList("Private-Token", pat.password()))) .orElseGet(() -> new RestRequest(baseApi)); var urlPattern = URIBuilder.base(gitLabHost.getUri()) @@ -136,7 +137,7 @@ public URI url() { var builder = URIBuilder .base(gitLabHost.getUri()) .setPath("/" + projectName + ".git"); - gitLabHost.getPat().ifPresent(pat -> builder.setAuthentication(pat.userName() + ":" + pat.token())); + gitLabHost.getPat().ifPresent(pat -> builder.setAuthentication(pat.username() + ":" + pat.password())); return builder.build(); } @@ -234,7 +235,7 @@ public HostedRepository fork() { e.printStackTrace(); } } - return gitLabHost.repository(forkedRepoName); + return gitLabHost.repository(forkedRepoName).orElseThrow(RuntimeException::new); } @Override diff --git a/forge/src/test/java/org/openjdk/skara/forge/ForgeTests.java b/forge/src/test/java/org/openjdk/skara/forge/ForgeTests.java new file mode 100644 index 000000000..63d7065eb --- /dev/null +++ b/forge/src/test/java/org/openjdk/skara/forge/ForgeTests.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.forge; + +import org.junit.jupiter.api.Test; +import org.openjdk.skara.host.Credential; +import org.openjdk.skara.json.JSONObject; + +import java.net.URI; +import java.util.*; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ForgeTests { + @Test + void sortTest() { + var allFactories = List.of(new ForgeFactory() { + @Override + public String name() { + return "something"; + } + + @Override + public Forge create(URI uri, Credential credential, JSONObject configuration) { + return null; + } + }, + new ForgeFactory() { + @Override + public String name() { + return "other"; + } + + @Override + public Forge create(URI uri, Credential credential, JSONObject configuration) { + return null; + } + }); + + var sorted = allFactories.stream() + .sorted(Comparator.comparing(f -> !f.name().contains("other"))) + .collect(Collectors.toList()); + + assertEquals("something", allFactories.get(0).name()); + assertEquals("other", sorted.get(0).name()); + } +} diff --git a/forge/src/test/java/org/openjdk/skara/forge/GitHubHostTests.java b/forge/src/test/java/org/openjdk/skara/forge/GitHubHostTests.java deleted file mode 100644 index a94402354..000000000 --- a/forge/src/test/java/org/openjdk/skara/forge/GitHubHostTests.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package org.openjdk.skara.forge; - -import org.openjdk.skara.network.URIBuilder; -import org.openjdk.skara.test.TemporaryDirectory; - -import org.junit.jupiter.api.*; - -import java.io.IOException; -import java.net.*; -import java.nio.charset.StandardCharsets; -import java.nio.file.*; -import java.util.regex.Pattern; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class GitHubHostTests { - private void generateKeyfile(Path path) throws IOException { - // This key was randomly generated for this test only - Files.writeString(path, "-----BEGIN PRIVATE KEY-----\n" + - "MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDAFzH+URXAvOoL\n" + - "0NSdIePQTTVsan13c+7D9tAilJAtRcxUjOz2lMZYBzrdsVYGbktfseEvF6o9dyoX\n" + - "X/py6DM0QqBNW/0uEv1ouS44po0VvykHVXrAq0u8E8HHFtr09VQSO/ceXrFd6haQ\n" + - "aCckbdp1TPn1q8w+U2bRkqUji7zzfwll6AaB4dhKZ1v5NFuff1PWmuk2x7b0u2yR\n" + - "uANLHLqmNB4ik7bUTiIyacXeVSZRZRFGwJjd+1WnyiybwV6QbQ0nndw6iaz2wGWt\n" + - "XDif7DJE0axMReUZVKJLqMagS5R5ra6FdlnUPw0nbJMwnDOLk9ofSfne0LTSTD6K\n" + - "/VZ26izbAgMBAAECggEBALF0vDq1reLgo1dHFSQUquFEcpY1yrMP5wQifyVzGb65\n" + - "PIrfpgomZxXrl/Y2XcKTIg7FxcI7moouDDSL9lMxMByXcIAG+14VLQYSDSFIvA3b\n" + - "C4w666wSk2Ss29eQxbaG7aPqweDMmg6osy+1CHQfCDJVapYKoCTz54i0cNrlvSk0\n" + - "FZ3o99uAvAcLtrsqbnXO57NXQVajoSH0bkMZd+TuZqEIX3CzHoNEVhzvqaKedqA6\n" + - "Cd22Y2m6cnW0H10Chko05FtskLYD+jw275LiUtInplBtG3n5/uDIamsOPo9XG8i0\n" + - "a4rxaJYsRqXYqDOEjLi/QCUrYBtJ+gqT/qMOTjAoKAECgYEA/VPdvc03vScjIu4T\n" + - "vNXjXxv81HZPm/IoTYTgvTvrEqErQ/CIwTQJer1XUJ9M43n+XkVZsMKkUIMlwt2+\n" + - "G0wBwYkDUgIXFEJhb170BVgwyZHE+Djr0E7NunbAv/oQu8AfQzk5HZpcQwxVg8w8\n" + - "Vj2ecLb4GK0D9iJ4zLwlsRw2RukCgYEAwh30AG7gq5y9Mj/BusuDvyNZZKjE/pJz\n" + - "HtC7a/OzOyr+Bpr2VBxVDeEFth22bd/a4ohv1QcwNAa2LzelNfQRQURq/vqpDmuj\n" + - "g0ESQavh3i3Tax2LXO582HWueuNL+8Ufyb6WDJDvYuz0F3WBJhxixP3I7VgMhPWV\n" + - "tK/wEEDDwyMCgYEArR3M4NIHDzpZppsv3dIE6ZAEvWSEjrtzk1YFBwyVXkvJd0o/\n" + - "Clj3SWtu6eeS8bkCfYXC/ypkg6i7+2jxa1ILuShanoZTI0Mhtqwa8jQMUxNMmZy8\n" + - "ecQAjzZsDkVjfgqS0quePn6oIiGhpsnBSeYeCkTfUm2Z0XBJQRAqadgvt1ECgYBK\n" + - "FAgzyhxvIUeKT45s7JGAdcr9gPJ8fAL2tY1wqvWxFL0QZD6w5ocG3uLBFyGxWIY9\n" + - "gPe8ghvBHvaTmlav+k5DbAqw95Ngb29c/Y4sBZ4SncZa0FGIy3JVYMOPHgK3OAjj\n" + - "gpncfcr9I5QbB7qbgqWmq3rsKHfOnbHd3G5upWiPpQKBgCaPW2vyT/nfCvfh0z//\n" + - "QSv0//4zy7pDdOolP5ZRsUo5cU4aiv4XgTSglR2jEJyr4bMYCN/+4tnqp0tIUzt1\n" + - "RWJhXLU1dm4QhCTccWMAyQgktn3SB5Ww3+qyLr1klUwkO+rx8kkNjv3rC/u5EzQ9\n" + - "q3DJ9in4wyYBNPVDB5kJom5i\n" + - "-----END PRIVATE KEY-----", StandardCharsets.UTF_8); - } - - @Test - void webUriPatternReplacement() throws IOException, URISyntaxException { - try (var tempFolder = new TemporaryDirectory()) { - var key = tempFolder.path().resolve("key.pem"); - generateKeyfile(key); - var app = new GitHubApplication(key.toString(), "y", "z"); - var host = new GitHubHost(URIBuilder.base("http://www.example.com").build(), - app, Pattern.compile("^(http://www.example.com)/test/(.*)$"), "$1/another/$2"); - assertEquals(new URI("http://www.example.com/another/hello"), host.getWebURI("/test/hello")); - } - } -} diff --git a/forge/src/test/java/org/openjdk/skara/forge/PositionMapperTests.java b/forge/src/test/java/org/openjdk/skara/forge/PositionMapperTests.java deleted file mode 100644 index 4f1afd325..000000000 --- a/forge/src/test/java/org/openjdk/skara/forge/PositionMapperTests.java +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package org.openjdk.skara.forge; - -import org.junit.jupiter.api.*; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class PositionMapperTests { - private static final String diff = "diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/Range.java b/vcs/src/main/java/org/openjdk/skara/vcs/Range.java\n" + - "index d849c08..c42e24a 100644\n" + - "--- a/vcs/src/main/java/org/openjdk/skara/vcs/Range.java\n" + - "+++ b/vcs/src/main/java/org/openjdk/skara/vcs/Range.java\n" + - "@@ -42,18 +42,7 @@ public static Range fromString(String s) {\n" + - " }\n" + - " \n" + - " var start = Integer.parseInt(s.substring(0, separatorIndex));\n" + - "-\n" + - "- // Need to work arond a bug in git where git sometimes print -1\n" + - "- // as an unsigned int for the count part of the range\n" + - "- var countString = s.substring(separatorIndex + 1, s.length());\n" + - "- var count =\n" + - "- countString.equals(\"18446744073709551615\") ? 0 : Integer.parseInt(countString);\n" + - "-\n" + - "- if (count == 0 && start != 0) {\n" + - "- // start is off-by-one when count is 0.\n" + - "- // but if start == 0, a file was added and we need a 0 here.\n" + - "- start++;\n" + - "- }\n" + - "+ var count = Integer.parseInt(s.substring(separatorIndex + 1, s.length()));\n" + - " \n" + - " return new Range(start, count);\n" + - " }\n" + - "diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitCombinedDiffParser.java b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitCombinedDiffParser.java\n" + - "index f829554..8044ad1 100644\n" + - "--- a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitCombinedDiffParser.java\n" + - "+++ b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitCombinedDiffParser.java\n" + - "@@ -43,7 +43,7 @@ public GitCombinedDiffParser(List bases, Hash head, String delimiter) {\n" + - " this.delimiter = delimiter;\n" + - " }\n" + - " \n" + - "- private List> parseSingleFileMultiParentDiff(UnixStreamReader reader) throws IOException {\n" + - "+ private List> parseSingleFileMultiParentDiff(UnixStreamReader reader, List headers) throws IOException {\n" + - " assert line.startsWith(\"diff --combined\");\n" + - " \n" + - " while ((line = reader.readLine()) != null &&\n" + - "@@ -64,7 +64,14 @@ public GitCombinedDiffParser(List bases, Hash head, String delimiter) {\n" + - " assert words[0].startsWith(\"@@@\");\n" + - " var sourceRangesPerParent = new ArrayList(numParents);\n" + - " for (int i = 1; i <= numParents; i++) {\n" + - "- sourceRangesPerParent.add(Range.fromString(words[i].substring(1))); // skip initial '-'\n" + - "+ var header = headers.get(i - 1);\n" + - "+ if (header.status().isAdded()) {\n" + - "+ // git reports wrong start for added files, they should\n" + - "+ // always have range (0,0), but git reports (1,0)\n" + - "+ sourceRangesPerParent.add(new Range(0, 0));\n" + - "+ } else {\n" + - "+ sourceRangesPerParent.add(Range.fromString(words[i].substring(1))); // skip initial '-'\n" + - "+ }\n" + - " }\n" + - " var targetRange = Range.fromString(words[numParents + 1].substring(1)); // skip initial '+'\n" + - " \n" + - "@@ -174,8 +181,10 @@ public GitCombinedDiffParser(List bases, Hash head, String delimiter) {\n" + - " headersPerParent.add(new ArrayList());\n" + - " }\n" + - " \n" + - "+ var headersForFiles = new ArrayList>();\n" + - " while (line != null && line.startsWith(\"::\")) {\n" + - " var headersForFile = parseCombinedRawLine(line);\n" + - "+ headersForFiles.add(headersForFile);\n" + - " assert headersForFile.size() == numParents;\n" + - " \n" + - " for (int i = 0; i < numParents; i++) {\n" + - "@@ -193,13 +202,18 @@ public GitCombinedDiffParser(List bases, Hash head, String delimiter) {\n" + - " for (int i = 0; i < numParents; i++) {\n" + - " hunksPerFilePerParent.add(new ArrayList>());\n" + - " }\n" + - "+\n" + - "+ int headerIndex = 0;\n" + - " while (line != null && !line.equals(delimiter)) {\n" + - "- var hunksPerParentForFile = parseSingleFileMultiParentDiff(reader);\n" + - "+ var headersForFile = headersForFiles.get(headerIndex);\n" + - "+ var hunksPerParentForFile = parseSingleFileMultiParentDiff(reader, headersForFile);\n" + - " assert hunksPerParentForFile.size() == numParents;\n" + - " \n" + - " for (int i = 0; i < numParents; i++) {\n" + - " hunksPerFilePerParent.get(i).add(hunksPerParentForFile.get(i));\n" + - " }\n" + - "+\n" + - "+ headerIndex++;\n" + - " }\n" + - " \n" + - " var patchesPerParent = new ArrayList>(numParents);\n" + - "diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/tools/GitRange.java b/vcs/src/main/java/org/openjdk/skara/vcs/tools/GitRange.java\n" + - "new file mode 100644\n" + - "index 0000000..62a6bde\n" + - "--- /dev/null\n" + - "+++ b/vcs/src/main/java/org/openjdk/skara/vcs/tools/GitRange.java\n" + - "@@ -0,0 +1,52 @@\n" + - "+/*\n" + - "+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.\n" + - "+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.\n" + - "+ *\n" + - "+ * This code is free software; you can redistribute it and/or modify it\n" + - "+ * under the terms of the GNU General Public License version 2 only, as\n" + - "+ * published by the Free Software Foundation.\n" + - "+ *\n" + - "+ * This code is distributed in the hope that it will be useful, but WITHOUT\n" + - "+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or\n" + - "+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License\n" + - "+ * version 2 for more details (a copy is included in the LICENSE file that\n" + - "+ * accompanied this code).\n" + - "+ *\n" + - "+ * You should have received a copy of the GNU General Public License version\n" + - "+ * 2 along with this work; if not, write to the Free Software Foundation,\n" + - "+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.\n" + - "+ *\n" + - "+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA\n" + - "+ * or visit www.oracle.com if you need additional information or have any\n" + - "+ * questions.\n" + - "+ */\n" + - "+package org.openjdk.skara.vcs.tools;\n" + - "+\n" + - "+import org.openjdk.skara.vcs.Range;\n" + - "+\n" + - "+class GitRange {\n" + - "+ static Range fromString(String s) {\n" + - "+ var separatorIndex = s.indexOf(\",\");\n" + - "+\n" + - "+ if (separatorIndex == -1) {\n" + - "+ var start = Integer.parseInt(s);\n" + - "+ return new Range(start, 1);\n" + - "+ }\n" + - "+\n" + - "+ var start = Integer.parseInt(s.substring(0, separatorIndex));\n" + - "+\n" + - "+ // Need to work around a bug in git where git sometimes print -1\n" + - "+ // as an unsigned int for the count part of the range\n" + - "+ var countString = s.substring(separatorIndex + 1, s.length());\n" + - "+ var count =\n" + - "+ countString.equals(\"18446744073709551615\") ? 0 : Integer.parseInt(countString);\n" + - "+\n" + - "+ if (count == 0 && start != 0) {\n" + - "+ // start is off-by-one when count is 0.\n" + - "+ // but if start == 0, a file was added and we need a 0 here.\n" + - "+ start++;\n" + - "+ }\n" + - "+\n" + - "+ return new Range(start, count);\n" + - "+ }\n" + - "+}\n" + - "diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/tools/UnifiedDiffParser.java b/vcs/src/main/java/org/openjdk/skara/vcs/tools/UnifiedDiffParser.java\n" + - "index 2bf6972..2dbeccd 100644\n" + - "--- a/vcs/src/main/java/org/openjdk/skara/vcs/tools/UnifiedDiffParser.java\n" + - "+++ b/vcs/src/main/java/org/openjdk/skara/vcs/tools/UnifiedDiffParser.java\n" + - "@@ -149,8 +149,8 @@ private Hunks parseSingleFileTextualHunks(UnixStreamReader reader) throws IOExce\n" + - " throw new IllegalStateException(\"Unexpected diff line: \" + line);\n" + - " }\n" + - " }\n" + - "- hunks.add(new Hunk(Range.fromString(sourceRange), sourceLines, sourceHasNewlineAtEndOfFile,\n" + - "- Range.fromString(targetRange), targetLines, targetHasNewlineAtEndOfFile));\n" + - "+ hunks.add(new Hunk(GitRange.fromString(sourceRange), sourceLines, sourceHasNewlineAtEndOfFile,\n" + - "+ GitRange.fromString(targetRange), targetLines, targetHasNewlineAtEndOfFile));\n" + - " }\n" + - " \n" + - " return Hunks.ofTextual(hunks);\n" + - "@@ -261,14 +261,6 @@ private Hunks parseSingleFileHunks(UnixStreamReader reader) throws IOException {\n" + - " }\n" + - " \n" + - " if (line.startsWith(\" \")) {\n" + - "- // this is the start of another hunk\n" + - "- // TODO: explain this strange behaviour\n" + - "- if (sourceLines.size() == 0) {\n" + - "- sourceStart--;\n" + - "- }\n" + - "- if (targetLines.size() == 0) {\n" + - "- targetStart--;\n" + - "- }\n" + - " hunks.add(new Hunk(new Range(sourceStart, sourceLines.size()), sourceLines,\n" + - " new Range(targetStart, targetLines.size()), targetLines));\n" + - " \n" + - "@@ -287,12 +279,6 @@ private Hunks parseSingleFileHunks(UnixStreamReader reader) throws IOException {\n" + - " }\n" + - " \n" + - " if (sourceLines.size() > 0 || targetLines.size() > 0) {\n" + - "- if (sourceLines.size() == 0) {\n" + - "- sourceStart--;\n" + - "- }\n" + - "- if (targetLines.size() == 0) {\n" + - "- targetStart--;\n" + - "- }\n" + - " hunks.add(new Hunk(new Range(sourceStart, sourceLines.size()), sourceLines,\n" + - " new Range(targetStart, targetLines.size()), targetLines));\n" + - " }\n" + - "diff --git a/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java b/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java\n" + - "index 5d476f1..8747062 100644\n" + - "--- a/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java\n" + - "+++ b/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java\n" + - "@@ -376,7 +376,7 @@ void testCommitListingWithMultipleCommits(VCS vcs) throws IOException {\n" + - " assertEquals(1, hunks.size());\n" + - " \n" + - " var hunk = hunks.get(0);\n" + - "- assertEquals(new Range(1, 0), hunk.source().range());\n" + - "+ assertEquals(new Range(2, 0), hunk.source().range());\n" + - " assertEquals(new Range(2, 1), hunk.target().range());\n" + - " \n" + - " assertEquals(List.of(), hunk.source().lines());\n" + - "@@ -508,7 +508,7 @@ void testSquash(VCS vcs) throws IOException {\n" + - " assertEquals(1, hunks.size());\n" + - " \n" + - " var hunk = hunks.get(0);\n" + - "- assertEquals(new Range(1, 0), hunk.source().range());\n" + - "+ assertEquals(new Range(2, 0), hunk.source().range());\n" + - " assertEquals(new Range(2, 2), hunk.target().range());\n" + - " \n" + - " assertEquals(List.of(), hunk.source().lines());\n" + - "@@ -859,7 +859,7 @@ void testDiffBetweenCommits(VCS vcs) throws IOException {\n" + - " assertEquals(1, hunks.size());\n" + - " \n" + - " var hunk = hunks.get(0);\n" + - "- assertEquals(1, hunk.source().range().start());\n" + - "+ assertEquals(2, hunk.source().range().start());\n" + - " assertEquals(0, hunk.source().range().count());\n" + - " assertEquals(0, hunk.source().lines().size());\n" + - " \n" + - "@@ -1132,7 +1132,7 @@ void testDiffWithWorkingDir(VCS vcs) throws IOException {\n" + - " assertEquals(1, hunks.size());\n" + - " \n" + - " var hunk = hunks.get(0);\n" + - "- assertEquals(1, hunk.source().range().start());\n" + - "+ assertEquals(2, hunk.source().range().start());\n" + - " assertEquals(0, hunk.source().range().count());\n" + - " assertEquals(List.of(), hunk.source().lines());\n" + - " \n" + - "@@ -1283,7 +1283,7 @@ void testMergeWithEdit(VCS vcs) throws IOException {\n" + - " assertEquals(List.of(), secondHunk.source().lines());\n" + - " assertEquals(List.of(\"One last line\"), secondHunk.target().lines());\n" + - " \n" + - "- assertEquals(2, secondHunk.source().range().start());\n" + - "+ assertEquals(3, secondHunk.source().range().start());\n" + - " assertEquals(0, secondHunk.source().range().count());\n" + - " assertEquals(3, secondHunk.target().range().start());\n" + - " assertEquals(1, secondHunk.target().range().count());\n" + - "@@ -1302,7 +1302,7 @@ void testMergeWithEdit(VCS vcs) throws IOException {\n" + - " assertEquals(List.of(), thirdHunk.source().lines());\n" + - " assertEquals(List.of(\"One more line\", \"One last line\"), thirdHunk.target().lines());\n" + - " \n" + - "- assertEquals(1, thirdHunk.source().range().start());\n" + - "+ assertEquals(2, thirdHunk.source().range().start());\n" + - " assertEquals(0, thirdHunk.source().range().count());\n" + - " assertEquals(2, thirdHunk.target().range().start());\n" + - " assertEquals(2, thirdHunk.target().range().count());\n"; - - @Test - void simple() { - var mapper = PositionMapper.parse(diff); - - assertEquals(38, mapper.positionToLine("vcs/src/main/java/org/openjdk/skara/vcs/tools/GitRange.java", 38)); - assertEquals(70, mapper.positionToLine("vcs/src/main/java/org/openjdk/skara/vcs/git/GitCombinedDiffParser.java", 17)); - } -} diff --git a/forge/src/test/java/org/openjdk/skara/forge/GitHubApplicationTests.java b/forge/src/test/java/org/openjdk/skara/forge/github/GitHubApplicationTests.java similarity index 96% rename from forge/src/test/java/org/openjdk/skara/forge/GitHubApplicationTests.java rename to forge/src/test/java/org/openjdk/skara/forge/github/GitHubApplicationTests.java index c91dcca42..4481e753e 100644 --- a/forge/src/test/java/org/openjdk/skara/forge/GitHubApplicationTests.java +++ b/forge/src/test/java/org/openjdk/skara/forge/github/GitHubApplicationTests.java @@ -20,9 +20,9 @@ * or visit www.oracle.com if you need additional information or have any * questions. */ -package org.openjdk.skara.forge; +package org.openjdk.skara.forge.github; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.Test; import java.time.Duration; diff --git a/forge/src/test/java/org/openjdk/skara/forge/github/GitHubHostTests.java b/forge/src/test/java/org/openjdk/skara/forge/github/GitHubHostTests.java new file mode 100644 index 000000000..2af227644 --- /dev/null +++ b/forge/src/test/java/org/openjdk/skara/forge/github/GitHubHostTests.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.forge.github; + +import org.junit.jupiter.api.Test; +import org.openjdk.skara.network.URIBuilder; +import org.openjdk.skara.test.TemporaryDirectory; + +import java.io.IOException; +import java.net.*; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class GitHubHostTests { + @Test + void webUriPatternReplacement() throws IOException, URISyntaxException { + try (var tempFolder = new TemporaryDirectory()) { + var host = new GitHubHost(URIBuilder.base("http://www.example.com").build(), + Pattern.compile("^(http://www.example.com)/test/(.*)$"), "$1/another/$2"); + assertEquals(new URI("http://www.example.com/another/hello"), host.getWebURI("/test/hello")); + } + } +} diff --git a/gradlew b/gradlew index bb95ec1f3..b36faf64b 100644 --- a/gradlew +++ b/gradlew @@ -81,68 +81,71 @@ extract_zip() { } DIR=$(dirname $0) +ARCH=$(uname -m) OS=$(uname) . $(dirname "${0}")/deps.env -case "${OS}" in - Linux ) - JDK_URL="${JDK_LINUX_URL}" - JDK_SHA256="${JDK_LINUX_SHA256}" - ;; - Darwin ) - JDK_URL="${JDK_MACOS_URL}" - JDK_SHA256="${JDK_MACOS_SHA256}" - ;; - CYGWIN_NT* ) - JDK_URL="${JDK_WINDOWS_URL}" - JDK_SHA256="${JDK_WINDOWS_SHA256}" - ;; - *) - echo "error: unknown operating system ${OS}" - exit 1 - ;; -esac - -JDK_FILENAME="${DIR}/.jdk/$(basename ${JDK_URL})" -if [ "${OS}" = "Linux" -o "${OS}" = "Darwin" ]; then - JDK_DIR="${DIR}/.jdk/$(basename -s '.tar.gz' ${JDK_URL})" -else - JDK_DIR="${DIR}/.jdk/$(basename -s '.zip' ${JDK_URL})" +if [ "${ARCH}" = "x86_64" ]; then + case "${OS}" in + Linux ) + JDK_URL="${JDK_LINUX_X64_URL}" + JDK_SHA256="${JDK_LINUX_X64_SHA256}" + ;; + Darwin ) + JDK_URL="${JDK_MACOS_X64_URL}" + JDK_SHA256="${JDK_MACOS_X64_SHA256}" + ;; + CYGWIN_NT* ) + JDK_URL="${JDK_WINDOWS_X64_URL}" + JDK_SHA256="${JDK_WINDOWS_X64_SHA256}" + ;; + esac fi -if [ ! -d "${JDK_DIR}" ]; then - mkdir -p ${DIR}/.jdk - if [ ! -f "${JDK_FILENAME}" ]; then - if [ -f "${JDK_URL}" ]; then - echo "Copying JDK..." - cp "${JDK_URL}" "${JDK_FILENAME}" +if [ ! -z "${JDK_URL}" ]; then + JDK_FILENAME="${DIR}/.jdk/$(basename ${JDK_URL})" + if [ "${OS}" = "Linux" -o "${OS}" = "Darwin" ]; then + JDK_DIR="${DIR}/.jdk/$(basename -s '.tar.gz' ${JDK_URL})" + else + JDK_DIR="${DIR}/.jdk/$(basename -s '.zip' ${JDK_URL})" + fi + + if [ ! -d "${JDK_DIR}" ]; then + mkdir -p ${DIR}/.jdk + if [ ! -f "${JDK_FILENAME}" ]; then + if [ -f "${JDK_URL}" ]; then + echo "Copying JDK..." + cp "${JDK_URL}" "${JDK_FILENAME}" + else + echo "Downloading JDK..." + download ${JDK_URL} "${JDK_FILENAME}" + checksum "${JDK_FILENAME}" ${JDK_SHA256} + fi + fi + echo "Extracting JDK..." + if [ "${OS}" = "Linux" -o "${OS}" = "Darwin" ]; then + extract_tar "${JDK_FILENAME}" "${JDK_DIR}" else - echo "Downloading JDK..." - download ${JDK_URL} "${JDK_FILENAME}" - checksum "${JDK_FILENAME}" ${JDK_SHA256} + extract_zip "${JDK_FILENAME}" "${JDK_DIR}" fi fi - echo "Extracting JDK..." - if [ "${OS}" = "Linux" -o "${OS}" = "Darwin" ]; then - extract_tar "${JDK_FILENAME}" "${JDK_DIR}" + + if [ "${OS}" = "Darwin" ]; then + EXECUTABLE_FILTER='-perm +111' + LAUNCHER='java' + elif [ "${OS}" = "Linux" ]; then + EXECUTABLE_FILTER='-executable' + LAUNCHER='java' else - extract_zip "${JDK_FILENAME}" "${JDK_DIR}" + LAUNCHER='java.exe' fi -fi -if [ "${OS}" = "Darwin" ]; then - EXECUTABLE_FILTER='-perm +111' - LAUNCHER='java' -elif [ "${OS}" = "Linux" ]; then - EXECUTABLE_FILTER='-executable' - LAUNCHER='java' + JAVA_LAUNCHER=$(find "${JDK_DIR}" -type f ${EXECUTABLE_FILTER} | grep ".*/bin/${LAUNCHER}$") + export JAVA_HOME="$(dirname $(dirname ${JAVA_LAUNCHER}))" else - LAUNCHER='java.exe' + JAVA_LAUNCHER="java" fi -JAVA_LAUNCHER=$(find "${JDK_DIR}" -type f ${EXECUTABLE_FILTER} | grep ".*/bin/${LAUNCHER}$") -export JAVA_HOME="$(dirname $(dirname ${JAVA_LAUNCHER}))" - GRADLE_FILENAME="${DIR}/.gradle/$(basename ${GRADLE_URL})" GRADLE_DIR="${DIR}/.gradle/$(basename -s '.zip' ${GRADLE_URL})" @@ -155,7 +158,11 @@ if [ ! -d "${GRADLE_DIR}" ]; then checksum ${GRADLE_FILENAME} ${GRADLE_SHA256} echo "Extracting Gradle..." if [ "${OS}" = "Linux" -o "${OS}" = "Darwin" ]; then - "${JAVA_LAUNCHER}" "${DIR}"/Unzip.java "${GRADLE_FILENAME}" "${GRADLE_DIR}" + if exists unzip; then + extract_zip "${GRADLE_FILENAME}" "${GRADLE_DIR}" + else + "${JAVA_LAUNCHER}" "${DIR}"/Unzip.java "${GRADLE_FILENAME}" "${GRADLE_DIR}" + fi else extract_zip "${GRADLE_FILENAME}" "${GRADLE_DIR}" fi diff --git a/gradlew.bat b/gradlew.bat index f1842dd5e..3fa9fad29 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -21,17 +21,17 @@ rem or visit www.oracle.com if you need additional information or have any rem questions. for /f "tokens=1,2 delims==" %%A in (deps.env) do (set %%A=%%~B) -for /f %%i in ("%JDK_WINDOWS_URL%") do set JDK_WINDOWS_DIR=%%~ni +for /f %%i in ("%JDK_WINDOWS_X64_URL%") do set JDK_WINDOWS_DIR=%%~ni for /f %%i in ("%GRADLE_URL%") do set GRADLE_DIR=%%~ni if exist %~dp0\.jdk\%JDK_WINDOWS_DIR% goto gradle echo Downloading JDK... mkdir %~dp0\.jdk\temp -curl -L %JDK_WINDOWS_URL% -o %JDK_WINDOWS_DIR%.zip +curl -L %JDK_WINDOWS_X64_URL% -o %JDK_WINDOWS_DIR%.zip move %JDK_WINDOWS_DIR%.zip %~dp0\.jdk\ for /f "tokens=*" %%i in ('@certutil -hashfile %~dp0/.jdk/%JDK_WINDOWS_DIR%.zip sha256 ^| %WINDIR%\System32\find /v "hash of file" ^| %WINDIR%\System32\find /v "CertUtil"') do set SHA256JDK=%%i -if "%SHA256JDK%" == "%JDK_WINDOWS_SHA256%" (goto extractJdk) +if "%SHA256JDK%" == "%JDK_WINDOWS_X64_SHA256%" (goto extractJdk) echo Invalid SHA256 for JDK detected (%SHA256JDK%) goto done diff --git a/host/src/main/java/org/openjdk/skara/host/PersonalAccessToken.java b/host/src/main/java/org/openjdk/skara/host/Credential.java similarity index 76% rename from host/src/main/java/org/openjdk/skara/host/PersonalAccessToken.java rename to host/src/main/java/org/openjdk/skara/host/Credential.java index 37e1d9b04..254a8fad3 100644 --- a/host/src/main/java/org/openjdk/skara/host/PersonalAccessToken.java +++ b/host/src/main/java/org/openjdk/skara/host/Credential.java @@ -22,20 +22,20 @@ */ package org.openjdk.skara.host; -public class PersonalAccessToken { - private final String userName; - private final String token; +public class Credential { + private final String username; + private final String password; - public PersonalAccessToken(String userName, String token) { - this.userName = userName; - this.token = token; + public Credential(String username, String password) { + this.username = username; + this.password = password; } - public String token() { - return token; + public String password() { + return password; } - public String userName() { - return userName; + public String username() { + return username; } } diff --git a/issuetracker/src/main/java/module-info.java b/issuetracker/src/main/java/module-info.java index c9e3274b5..88ca7bf30 100644 --- a/issuetracker/src/main/java/module-info.java +++ b/issuetracker/src/main/java/module-info.java @@ -33,4 +33,8 @@ requires java.logging; exports org.openjdk.skara.issuetracker; + + uses org.openjdk.skara.issuetracker.IssueTrackerFactory; + + provides org.openjdk.skara.issuetracker.IssueTrackerFactory with org.openjdk.skara.issuetracker.jira.JiraIssueTrackerFactory; } diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/Comment.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/Comment.java index 8cf8cd650..3ffed6b2d 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/Comment.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/Comment.java @@ -27,7 +27,6 @@ import java.time.ZonedDateTime; public class Comment { - private final String id; private final String body; private final HostUser author; diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTracker.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTracker.java index e60e9d8e0..0e75f5c40 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTracker.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTracker.java @@ -22,8 +22,21 @@ */ package org.openjdk.skara.issuetracker; -import org.openjdk.skara.host.Host; +import org.openjdk.skara.host.*; +import org.openjdk.skara.json.JSONObject; + +import java.net.URI; public interface IssueTracker extends Host { IssueProject project(String name); + + static IssueTracker from(String name, URI uri, Credential credential, JSONObject configuration) { + var factory = IssueTrackerFactory.getIssueTrackerFactories().stream() + .filter(f -> f.name().equals(name)) + .findFirst(); + if (factory.isEmpty()) { + throw new RuntimeException("No issue tracker factory named '" + name + "' found - check module path"); + } + return factory.get().create(uri, credential, configuration); + } } diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTrackerFactory.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTrackerFactory.java index ef8358fd4..c4627012c 100644 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTrackerFactory.java +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/IssueTrackerFactory.java @@ -22,15 +22,28 @@ */ package org.openjdk.skara.issuetracker; -import org.openjdk.skara.host.*; +import org.openjdk.skara.host.Credential; +import org.openjdk.skara.json.JSONObject; import java.net.URI; +import java.util.*; +import java.util.stream.*; -public class IssueTrackerFactory { - public static IssueTracker createJiraHost(URI uri, PersonalAccessToken pat) { - if (pat != null) { - throw new RuntimeException("authentication not implemented yet"); - } - return new JiraHost(uri); +public interface IssueTrackerFactory { + /** + * A user-friendly name for the given issue tracker, used for configuration section naming. Should be lower case. + * @return + */ + String name(); + + /** + * Instantiate an instance of this issue tracker. + * @return + */ + IssueTracker create(URI uri, Credential credential, JSONObject configuration); + + static List getIssueTrackerFactories() { + return StreamSupport.stream(ServiceLoader.load(IssueTrackerFactory.class).spliterator(), false) + .collect(Collectors.toList()); } } diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraIssue.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraIssue.java deleted file mode 100644 index 76d44a3f6..000000000 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraIssue.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package org.openjdk.skara.issuetracker; - -import org.openjdk.skara.host.*; -import org.openjdk.skara.network.*; -import org.openjdk.skara.json.JSONValue; - -import java.net.URI; -import java.time.ZonedDateTime; -import java.util.List; - -public class JiraIssue implements Issue { - private final JiraProject jiraProject; - private final RestRequest request; - private final JSONValue json; - - JiraIssue(JiraProject jiraProject, RestRequest request, JSONValue json) { - this.jiraProject = jiraProject; - this.request = request; - this.json = json; - } - - @Override - public IssueProject project() { - return jiraProject; - } - - @Override - public String id() { - return json.get("key").asString(); - } - - @Override - public HostUser author() { - return new HostUser(json.get("fields").get("creator").get("key").asString(), - json.get("fields").get("creator").get("name").asString(), - json.get("fields").get("creator").get("displayName").asString()); - } - - @Override - public String title() { - return json.get("fields").get("summary").asString(); - } - - @Override - public void setTitle(String title) { - throw new RuntimeException("not implemented yet"); - } - - @Override - public String body() { - if (json.get("fields").get("description").isNull()) { - return ""; - } else { - return json.get("fields").get("description").asString(); - } - } - - @Override - public void setBody(String body) { - throw new RuntimeException("not implemented yet"); - } - - @Override - public List comments() { - throw new RuntimeException("not implemented yet"); - } - - @Override - public Comment addComment(String body) { - throw new RuntimeException("not implemented yet"); - } - - @Override - public Comment updateComment(String id, String body) { - throw new RuntimeException("not implemented yet"); - } - - @Override - public ZonedDateTime createdAt() { - return ZonedDateTime.parse(json.get("fields").get("created").asString()); - } - - @Override - public ZonedDateTime updatedAt() { - return ZonedDateTime.parse(json.get("fields").get("updated").asString()); - } - - @Override - public void setState(State state) { - throw new RuntimeException("not implemented yet"); - } - - @Override - public void addLabel(String label) { - throw new RuntimeException("not implemented yet"); - } - - @Override - public void removeLabel(String label) { - throw new RuntimeException("not implemented yet"); - } - - @Override - public List labels() { - throw new RuntimeException("not implemented yet"); - } - - @Override - public URI webUrl() { - return URIBuilder.base(jiraProject.webUrl()) - .setPath("/browse/" + id()) - .build(); - } - - @Override - public List assignees() { - throw new RuntimeException("not implemented yet"); - } - - @Override - public void setAssignees(List assignees) { - throw new RuntimeException("not implemented yet"); - } -} diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraProject.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraProject.java deleted file mode 100644 index dc9fe137e..000000000 --- a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/JiraProject.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package org.openjdk.skara.issuetracker; - -import org.openjdk.skara.json.JSON; -import org.openjdk.skara.network.*; - -import java.net.URI; -import java.util.*; - -public class JiraProject implements IssueProject { - private final JiraHost jiraHost; - private final String projectName; - private final RestRequest request; - - JiraProject(JiraHost host, RestRequest request, String projectName) { - this.jiraHost = host; - this.projectName = projectName; - this.request = request; - } - - @Override - public IssueTracker issueTracker() { - return jiraHost; - } - - @Override - public URI webUrl() { - return URIBuilder.base(jiraHost.getUri()).setPath("/projects/" + projectName).build(); - } - - @Override - public Issue createIssue(String title, List body) { - throw new RuntimeException("needs authentication; not implemented yet"); - } - - @Override - public Optional issue(String id) { - if (id.indexOf('-') < 0) { - id = projectName.toUpperCase() + "-" + id; - } - var issue = request.get("issue/" + id) - .onError(r -> r.statusCode() == 404 ? JSON.object().put("NOT_FOUND", true) : null) - .execute(); - if (!issue.contains("NOT_FOUND")) { - return Optional.of(new JiraIssue(this, request, issue)); - } else { - return Optional.empty(); - } - } - - @Override - public List issues() { - var ret = new ArrayList(); - var issues = request.post("search") - .body("jql", "project = " + projectName + " AND status in (Open, New)") - .execute(); - for (var issue : issues.get("issues").asArray()) { - ret.add(new JiraIssue(this, request, issue)); - } - return ret; - } -} diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraHost.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraHost.java new file mode 100644 index 000000000..a18ded303 --- /dev/null +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraHost.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.issuetracker.jira; + +import org.openjdk.skara.host.HostUser; +import org.openjdk.skara.issuetracker.*; +import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; + +import java.net.URI; +import java.util.Arrays; + +public class JiraHost implements IssueTracker { + private final URI uri; + private final RestRequest request; + + JiraHost(URI uri) { + this.uri = uri; + + var baseApi = URIBuilder.base(uri) + .setPath("/rest/api/2/") + .build(); + request = new RestRequest(baseApi); + } + + JiraHost(URI uri, JiraVault jiraVault) { + this.uri = uri; + var baseApi = URIBuilder.base(uri) + .setPath("/rest/api/2/") + .build(); + request = new RestRequest(baseApi, () -> Arrays.asList("Cookie", jiraVault.getCookie())); + } + + URI getUri() { + return uri; + } + + @Override + public boolean isValid() { + var version = request.get("serverInfo") + .onError(r -> JSON.object().put("invalid", true)) + .execute(); + return !version.contains("invalid"); + } + + @Override + public IssueProject project(String name) { + return new JiraProject(this, request, name); + } + + private JSONObject userData(String name) { + var data = request.get("user") + .param("username", name) + .execute(); + return data.asObject(); + } + + @Override + public HostUser user(String username) { + var data = request.get("user") + .param("username", username) + .execute(); + var user = new HostUser(data.get("name").asString(), + data.get("name").asString(), + data.get("displayName").asString()); + return user; + } + + @Override + public HostUser currentUser() { + var data = request.get("myself").execute(); + var user = new HostUser(data.get("name").asString(), + data.get("name").asString(), + data.get("displayName").asString()); + return user; + } + + @Override + public boolean isMemberOf(String groupId, HostUser user) { + var data = request.get("user") + .param("username", user.id()) + .param("expand", "groups") + .execute(); + for (var group : data.get("groups").get("items").asArray()) { + if (group.get("name").asString().equals(groupId)) { + return true; + } + } + return false; + } +} diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssue.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssue.java new file mode 100644 index 000000000..545ea2ed0 --- /dev/null +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssue.java @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.issuetracker.jira; + +import org.openjdk.skara.host.*; +import org.openjdk.skara.issuetracker.*; +import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; + +import java.net.URI; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.stream.Collectors; + +public class JiraIssue implements Issue { + private final JiraProject jiraProject; + private final RestRequest request; + private final JSONValue json; + + private static final DateTimeFormatter dateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + + JiraIssue(JiraProject jiraProject, RestRequest request, JSONValue json) { + this.jiraProject = jiraProject; + this.request = request; + this.json = json; + } + + @Override + public IssueProject project() { + return jiraProject; + } + + @Override + public String id() { + return json.get("key").asString(); + } + + @Override + public HostUser author() { + return new HostUser(json.get("fields").get("creator").get("key").asString(), + json.get("fields").get("creator").get("name").asString(), + json.get("fields").get("creator").get("displayName").asString()); + } + + @Override + public String title() { + return json.get("fields").get("summary").asString(); + } + + @Override + public void setTitle(String title) { + var query = JSON.object() + .put("fields", JSON.object() + .put("summary", title)); + request.put("").body(query).execute(); + } + + @Override + public String body() { + if (json.get("fields").get("description").isNull()) { + return ""; + } else { + return json.get("fields").get("description").asString(); + } + } + + @Override + public void setBody(String body) { + var query = JSON.object() + .put("fields", JSON.object() + .put("description", body)); + request.put("").body(query).execute(); + } + + private Comment parseComment(JSONValue json) { + return new Comment(json.get("id").asString(), + json.get("body").asString(), + new HostUser(json.get("author").get("name").asString(), + json.get("author").get("name").asString(), + json.get("author").get("displayName").asString()), + ZonedDateTime.parse(json.get("created").asString(), dateFormat), + ZonedDateTime.parse(json.get("updated").asString(), dateFormat)); + } + + @Override + public List comments() { + var comments = request.get("/comment") + .param("maxResults", "1000") + .execute(); + return comments.get("comments").stream() + .map(this::parseComment) + .collect(Collectors.toList()); + } + + @Override + public Comment addComment(String body) { + var json = request.post("/comment") + .body("body", body) + .execute(); + return parseComment(json); + } + + @Override + public Comment updateComment(String id, String body) { + var json = request.put("/comment/" + id) + .body("body", body) + .execute(); + return parseComment(json); + } + + @Override + public ZonedDateTime createdAt() { + return ZonedDateTime.parse(json.get("fields").get("created").asString(), dateFormat); + } + + @Override + public ZonedDateTime updatedAt() { + return ZonedDateTime.parse(json.get("fields").get("updated").asString(), dateFormat); + } + + @Override + public void setState(State state) { + var transitions = request.get("/transitions").execute(); + var wantedStateName = state == State.CLOSED ? "Closed" : "Open"; + for (var transition : transitions.get("transitions").asArray()) { + if (transition.get("to").get("name").asString().equals(wantedStateName)) { + var query = JSON.object() + .put("transition", JSON.object() + .put("id", transition.get("id").asString())); + request.post("/transitions") + .body(query) + .execute(); + return; + } + } + } + + @Override + public void addLabel(String label) { + var query = JSON.object() + .put("update", JSON.object() + .put("labels", JSON.array().add(JSON.object() + .put("add", label)))); + request.put("").body(query).execute(); + } + + @Override + public void removeLabel(String label) { + var query = JSON.object() + .put("update", JSON.object() + .put("labels", JSON.array().add(JSON.object() + .put("remove", label)))); + request.put("").body(query).execute(); + } + + @Override + public List labels() { + return json.get("fields").get("labels").stream() + .map(JSONValue::asString) + .collect(Collectors.toList()); + } + + @Override + public URI webUrl() { + return URIBuilder.base(jiraProject.webUrl()) + .setPath("/browse/" + id()) + .build(); + } + + @Override + public List assignees() { + var assignee = json.get("fields").get("assignee"); + if (assignee.isNull()) { + return List.of(); + } + + var user = new HostUser(assignee.get("name").asString(), + assignee.get("name").asString(), + assignee.get("displayName").asString()); + return List.of(user); + } + + @Override + public void setAssignees(List assignees) { + String assignee; + switch (assignees.size()) { + case 0: + assignee = null; + break; + case 1: + assignee = assignees.get(0).id(); + break; + default: + throw new RuntimeException("multiple assignees not supported"); + } + request.put("/assignee") + .body("name", assignee) + .execute(); + } +} diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssueTrackerFactory.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssueTrackerFactory.java new file mode 100644 index 000000000..c184e7ead --- /dev/null +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraIssueTrackerFactory.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.issuetracker.jira; + +import org.openjdk.skara.host.Credential; +import org.openjdk.skara.issuetracker.*; +import org.openjdk.skara.json.JSONObject; +import org.openjdk.skara.network.URIBuilder; + +import java.net.URI; + +public class JiraIssueTrackerFactory implements IssueTrackerFactory { + @Override + public String name() { + return "jira"; + } + + @Override + public IssueTracker create(URI uri, Credential credential, JSONObject configuration) { + if (credential == null) { + return new JiraHost(uri); + } else { + if (credential.username().startsWith("https://")) { + var vaultUrl = URIBuilder.base(credential.username()).build(); + var jiraVault = new JiraVault(vaultUrl, credential.password()); + return new JiraHost(uri, jiraVault); + } else { + throw new RuntimeException("basic authentication not implemented yet"); + } + } + } +} diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java new file mode 100644 index 000000000..bff9e6457 --- /dev/null +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraProject.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.issuetracker.jira; + +import org.openjdk.skara.issuetracker.*; +import org.openjdk.skara.json.*; +import org.openjdk.skara.network.*; + +import java.net.URI; +import java.util.*; + +public class JiraProject implements IssueProject { + private final JiraHost jiraHost; + private final String projectName; + private final RestRequest request; + + private JSONObject projectMetadataCache = null; + + JiraProject(JiraHost host, RestRequest request, String projectName) { + this.jiraHost = host; + this.projectName = projectName; + this.request = request; + } + + private JSONObject project() { + if (projectMetadataCache == null) { + projectMetadataCache = request.get("project/" + projectName).execute().asObject(); + } + return projectMetadataCache; + } + + private Map issueTypes() { + var ret = new HashMap(); + for (var type : project().get("issueTypes").asArray()) { + ret.put(type.get("name").asString(), type.get("id").asString()); + } + return ret; + } + + private Map components() { + var ret = new HashMap(); + for (var type : project().get("components").asArray()) { + ret.put(type.get("name").asString(), type.get("id").asString()); + } + return ret; + } + + private String projectId() { + return project().get("id").asString(); + } + + private String defaultIssueType() { + return issueTypes().values().stream() + .min(Comparator.naturalOrder()).orElseThrow(); + } + + private String defaultComponent() { + return components().values().stream() + .min(Comparator.naturalOrder()).orElseThrow(); + } + + @Override + public IssueTracker issueTracker() { + return jiraHost; + } + + @Override + public URI webUrl() { + return URIBuilder.base(jiraHost.getUri()).setPath("/projects/" + projectName).build(); + } + + @Override + public Issue createIssue(String title, List body) { + var query = JSON.object() + .put("fields", JSON.object() + .put("project", JSON.object() + .put("id", projectId())) + .put("issuetype", JSON.object() + .put("id", defaultIssueType())) + .put("components", JSON.array() + .add(JSON.object().put("id", defaultComponent()))) + .put("summary", title) + .put("description", String.join("\n", body))); + + var data = request.post("issue") + .body(query) + .execute(); + + return issue(data.get("key").asString()).orElseThrow(); + } + + @Override + public Optional issue(String id) { + if (id.indexOf('-') < 0) { + id = projectName.toUpperCase() + "-" + id; + } + var issueRequest = request.restrict("issue/" + id); + var issue = issueRequest.get("") + .onError(r -> r.statusCode() < 500 ? JSON.object().put("NOT_FOUND", true) : null) + .execute(); + if (issue == null) { + throw new RuntimeException("Server error when trying to fetch issue " + id); + } + if (!issue.contains("NOT_FOUND")) { + return Optional.of(new JiraIssue(this, issueRequest, issue)); + } else { + return Optional.empty(); + } + } + + @Override + public List issues() { + var ret = new ArrayList(); + var issues = request.post("search") + .body("jql", "project = " + projectName + " AND status in (Open, New)") + .execute(); + for (var issue : issues.get("issues").asArray()) { + ret.add(new JiraIssue(this, request, issue)); + } + return ret; + } +} diff --git a/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraVault.java b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraVault.java new file mode 100644 index 000000000..22fd95da2 --- /dev/null +++ b/issuetracker/src/main/java/org/openjdk/skara/issuetracker/jira/JiraVault.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.issuetracker.jira; + +import org.openjdk.skara.network.RestRequest; + +import java.net.URI; +import java.time.*; +import java.util.Arrays; +import java.util.logging.Logger; + +class JiraVault { + private final RestRequest request; + private final Logger log = Logger.getLogger("org.openjdk.skara.issuetracker.jira"); + + private String cookie; + private Instant expires; + + JiraVault(URI vaultUri, String vaultToken) { + request = new RestRequest(vaultUri, () -> Arrays.asList( + "X-Vault-Token", vaultToken + )); + } + + String getCookie() { + if ((cookie == null) || Instant.now().isAfter(expires)) { + var result = request.get("").execute(); + cookie = result.get("data").get("cookie.name").asString() + "=" + result.get("data").get("cookie.value").asString(); + expires = Instant.now().plus(Duration.ofSeconds(result.get("lease_duration").asInt()).dividedBy(2)); + log.info("Renewed Jira token (" + cookie + ") - expires " + expires); + } + return cookie; + } +} diff --git a/issuetracker/src/test/java/org/openjdk/skara/issuetracker/IssueTrackerTests.java b/issuetracker/src/test/java/org/openjdk/skara/issuetracker/IssueTrackerTests.java index f53df2d83..a206d9942 100644 --- a/issuetracker/src/test/java/org/openjdk/skara/issuetracker/IssueTrackerTests.java +++ b/issuetracker/src/test/java/org/openjdk/skara/issuetracker/IssueTrackerTests.java @@ -27,8 +27,9 @@ import org.junit.jupiter.api.*; import java.io.IOException; +import java.util.List; -import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.*; class IssueTrackerTests { @Test @@ -39,4 +40,32 @@ void isMemberOfNegativeTests(TestInfo info) throws IOException { assertFalse(host.isMemberOf(madeUpGroupIdThatCannotContainTestMember, host.currentUser())); } } + + @Test + void simple(TestInfo info) throws IOException { + try (var credentials = new HostCredentials(info)) { + var project = credentials.getIssueProject(); + + var userName = project.issueTracker().currentUser().userName(); + var user = project.issueTracker().user(userName); + assertEquals(userName, user.userName()); + + var issue = credentials.createIssue(project, "Test issue"); + issue.setTitle("Updated title"); + issue.setBody("This is now the body"); + var comment = issue.addComment("This is a comment"); + issue.updateComment(comment.id(), "Now it is updated"); + issue.addLabel("label"); + issue.addLabel("another"); + issue.removeLabel("label"); + issue.setAssignees(List.of(project.issueTracker().currentUser())); + + var updated = project.issue(issue.id()).orElseThrow(); + assertEquals(List.of("another"), updated.labels()); + assertEquals(List.of(project.issueTracker().currentUser()), updated.assignees()); + assertEquals(1, updated.comments().size()); + assertEquals("Updated title", updated.title()); + assertEquals("Now it is updated", updated.comments().get(0).body()); + } + } } diff --git a/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java b/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java index 12f89ea0b..c267de82f 100644 --- a/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java +++ b/jcheck/src/test/java/org/openjdk/skara/jcheck/TestRepository.java @@ -189,10 +189,18 @@ public Diff diff(Hash base, Hash head) throws IOException { return null; } + public Diff diff(Hash base, Hash head, List files) throws IOException { + return null; + } + public Diff diff(Hash head) throws IOException { return null; } + public Diff diff(Hash head, List files) throws IOException { + return null; + } + public List config(String key) throws IOException { return null; } @@ -232,4 +240,15 @@ public List remoteBranches(String remote) throws IOException { public List remotes() throws IOException { return null; } + + public void addSubmodule(String pullPath, Path path) throws IOException { + } + + public List submodules() throws IOException { + return null; + } + + public Optional annotate(Tag tag) throws IOException { + return null; + } } diff --git a/mailinglist/src/main/java/org/openjdk/skara/mailinglist/Mbox.java b/mailinglist/src/main/java/org/openjdk/skara/mailinglist/Mbox.java index ad9ac60c1..c680f5d55 100644 --- a/mailinglist/src/main/java/org/openjdk/skara/mailinglist/Mbox.java +++ b/mailinglist/src/main/java/org/openjdk/skara/mailinglist/Mbox.java @@ -42,7 +42,7 @@ public class Mbox { private final static Pattern fromStringEncodePattern = Pattern.compile("^(>*From )", Pattern.MULTILINE); private final static Pattern fromStringDecodePattern = Pattern.compile("^>(>*From )", Pattern.MULTILINE); - private static List splitMbox(String mbox) { + private static List splitMbox(String mbox, EmailAddress sender) { // Initial split var messages = mboxMessagePattern.matcher(mbox).results() .map(match -> match.group(1)) @@ -57,8 +57,11 @@ private static List splitMbox(String mbox) { for (var message : messages) { messageBuilder.insert(0, message); try { - var email = Email.parse(messageBuilder.toString()); - parsedMails.add(email); + var email = Email.from(Email.parse(messageBuilder.toString())); + if (sender != null) { + email.sender(sender); + } + parsedMails.add(email.build()); messageBuilder.setLength(0); } catch (RuntimeException ignored) { } @@ -79,7 +82,11 @@ private static String decodeFromStrings(String body) { } public static List parseMbox(String mbox) { - var emails = splitMbox(mbox); + return parseMbox(mbox, null); + } + + public static List parseMbox(String mbox, EmailAddress sender) { + var emails = splitMbox(mbox, sender); var idToMail = emails.stream().collect(Collectors.toMap(Email::id, Function.identity(), (a, b) -> a)); var idToConversation = idToMail.values().stream() .filter(email -> !email.hasHeader("In-Reply-To")) diff --git a/mailinglist/src/main/java/org/openjdk/skara/mailinglist/mailman/MailmanList.java b/mailinglist/src/main/java/org/openjdk/skara/mailinglist/mailman/MailmanList.java index c8cae0860..b2f4f1fd6 100644 --- a/mailinglist/src/main/java/org/openjdk/skara/mailinglist/mailman/MailmanList.java +++ b/mailinglist/src/main/java/org/openjdk/skara/mailinglist/mailman/MailmanList.java @@ -143,7 +143,7 @@ public List conversations(Duration maxAge) { if (newContent) { var concatenatedMbox = String.join("", actualPages); - var mails = Mbox.parseMbox(concatenatedMbox); + var mails = Mbox.parseMbox(concatenatedMbox, listAddress); var threshold = ZonedDateTime.now().minus(maxAge); cachedConversations = mails.stream() .filter(mail -> mail.first().date().isAfter(threshold)) diff --git a/mailinglist/src/test/java/org/openjdk/skara/mailinglist/MailmanTests.java b/mailinglist/src/test/java/org/openjdk/skara/mailinglist/MailmanTests.java index 623bf257e..f802673f9 100644 --- a/mailinglist/src/test/java/org/openjdk/skara/mailinglist/MailmanTests.java +++ b/mailinglist/src/test/java/org/openjdk/skara/mailinglist/MailmanTests.java @@ -45,12 +45,16 @@ void simple() throws IOException { .recipient(EmailAddress.parse(listAddress)) .build(); mailmanList.post(mail); + var expectedMail = Email.from(mail) + .sender(EmailAddress.parse(listAddress)) + .build(); + testServer.processIncoming(); var conversations = mailmanList.conversations(Duration.ofDays(1)); assertEquals(1, conversations.size()); var conversation = conversations.get(0); - assertEquals(mail, conversation.first()); + assertEquals(expectedMail, conversation.first()); } } @@ -67,11 +71,14 @@ void replies() throws IOException { .build(); mailmanList.post(sentParent); testServer.processIncoming(); + var expectedParent = Email.from(sentParent) + .sender(EmailAddress.parse(listAddress)) + .build(); var conversations = mailmanList.conversations(Duration.ofDays(1)); assertEquals(1, conversations.size()); var conversation = conversations.get(0); - assertEquals(sentParent, conversation.first()); + assertEquals(expectedParent, conversation.first()); var replier = EmailAddress.from("Replier", "replier@test.email"); var sentReply = Email.create(replier, "Reply subject", "Reply body") @@ -79,17 +86,21 @@ void replies() throws IOException { .header("In-Reply-To", sentParent.id().toString()) .build(); mailmanList.post(sentReply); + var expectedReply = Email.from(sentReply) + .sender(EmailAddress.parse(listAddress)) + .build(); + testServer.processIncoming(); conversations = mailmanList.conversations(Duration.ofDays(1)); assertEquals(1, conversations.size()); conversation = conversations.get(0); - assertEquals(sentParent, conversation.first()); + assertEquals(expectedParent, conversation.first()); var replies = conversation.replies(conversation.first()); assertEquals(1, replies.size()); var reply = replies.get(0); - assertEquals(sentReply, reply); + assertEquals(expectedReply, reply); } } @@ -107,18 +118,21 @@ void cached() throws IOException { mailmanList.post(mail); testServer.processIncoming(); + var expectedMail = Email.from(mail) + .sender(EmailAddress.parse(listAddress)) + .build(); { var conversations = mailmanList.conversations(Duration.ofDays(1)); assertEquals(1, conversations.size()); var conversation = conversations.get(0); - assertEquals(mail, conversation.first()); + assertEquals(expectedMail, conversation.first()); assertFalse(testServer.lastResponseCached()); } { var conversations = mailmanList.conversations(Duration.ofDays(1)); assertEquals(1, conversations.size()); var conversation = conversations.get(0); - assertEquals(mail, conversation.first()); + assertEquals(expectedMail, conversation.first()); assertTrue(testServer.lastResponseCached()); } } @@ -133,8 +147,8 @@ void interval() throws IOException { var mailmanList = mailmanServer.getList(listAddress); var sender = EmailAddress.from("Test", "test@test.email"); var mail1 = Email.create(sender, "Subject 1", "Body 1") - .recipient(EmailAddress.parse(listAddress)) - .build(); + .recipient(EmailAddress.parse(listAddress)) + .build(); var mail2 = Email.create(sender, "Subject 2", "Body 2") .recipient(EmailAddress.parse(listAddress)) .build(); @@ -142,13 +156,17 @@ void interval() throws IOException { mailmanList.post(mail1); mailmanList.post(mail2); }).start(); + var expectedMail = Email.from(mail1) + .sender(EmailAddress.parse(listAddress)) + .build(); + testServer.processIncoming(); assertThrows(RuntimeException.class, () -> testServer.processIncoming(Duration.ZERO)); var conversations = mailmanList.conversations(Duration.ofDays(1)); assertEquals(1, conversations.size()); var conversation = conversations.get(0); - assertEquals(mail1, conversation.first()); + assertEquals(expectedMail, conversation.first()); } } } diff --git a/network/src/main/java/org/openjdk/skara/network/RestRequest.java b/network/src/main/java/org/openjdk/skara/network/RestRequest.java index 3261563b0..977509bb9 100644 --- a/network/src/main/java/org/openjdk/skara/network/RestRequest.java +++ b/network/src/main/java/org/openjdk/skara/network/RestRequest.java @@ -24,7 +24,7 @@ import org.openjdk.skara.json.*; -import java.io.IOException; +import java.io.*; import java.net.URI; import java.net.http.*; import java.time.Duration; @@ -150,10 +150,14 @@ public QueryBuilder header(String name, String value) { } public JSONValue execute() { - return RestRequest.this.execute(this); + try { + return RestRequest.this.execute(this); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } - public String executeUnparsed() { + public String executeUnparsed() throws IOException { return RestRequest.this.executeUnparsed(this); } } @@ -216,7 +220,7 @@ void setRetryBackoffStep(Duration duration) { retryBackoffStep = duration; } - private HttpResponse sendRequest(HttpRequest request) { + private HttpResponse sendRequest(HttpRequest request) throws IOException { HttpResponse response; var retryCount = 0; @@ -227,14 +231,18 @@ private HttpResponse sendRequest(HttpRequest request) { .build(); response = client.send(request, HttpResponse.BodyHandlers.ofString()); break; - } catch (IOException | InterruptedException e) { + } catch (InterruptedException | IOException e) { if (retryCount < 5) { try { Thread.sleep(retryCount * retryBackoffStep.toMillis()); } catch (InterruptedException ignored) { } } else { - throw new RuntimeException(e); + try { + throw e; + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } } } retryCount++; @@ -297,7 +305,7 @@ private Map parseLink(String link) { .collect(Collectors.toMap(m -> m.group(2), m -> m.group(1))); } - private JSONValue execute(QueryBuilder queryBuilder) { + private JSONValue execute(QueryBuilder queryBuilder) throws IOException { var request = createRequest(queryBuilder.queryType, queryBuilder.endpoint, queryBuilder.composedBody(), queryBuilder.params, queryBuilder.headers); var response = sendRequest(request); @@ -339,10 +347,13 @@ private JSONValue execute(QueryBuilder queryBuilder) { return new JSONArray(ret.stream().flatMap(JSONArray::stream).toArray(JSONValue[]::new)); } - private String executeUnparsed(QueryBuilder queryBuilder) { + private String executeUnparsed(QueryBuilder queryBuilder) throws IOException { var request = createRequest(queryBuilder.queryType, queryBuilder.endpoint, queryBuilder.composedBody(), queryBuilder.params, queryBuilder.headers); var response = sendRequest(request); + if (response.statusCode() >= 400) { + throw new IOException("Bad response: " + response.statusCode()); + } return response.body(); } diff --git a/settings.gradle b/settings.gradle index 54bbef206..d2951b31f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -24,6 +24,7 @@ rootProject.name = 'skara' include 'args' include 'bot' +include 'ci' include 'cli' include 'census' include 'email' @@ -44,6 +45,7 @@ include 'network' include 'forge' include 'issuetracker' +include 'bots:bridgekeeper' include 'bots:cli' include 'bots:forward' include 'bots:hgbridge' @@ -53,4 +55,5 @@ include 'bots:mlbridge' include 'bots:notify' include 'bots:pr' include 'bots:submit' +include 'bots:tester' include 'bots:topological' diff --git a/storage/src/main/java/org/openjdk/skara/storage/HostedRepositoryStorage.java b/storage/src/main/java/org/openjdk/skara/storage/HostedRepositoryStorage.java index 2929f0c30..4668823b2 100644 --- a/storage/src/main/java/org/openjdk/skara/storage/HostedRepositoryStorage.java +++ b/storage/src/main/java/org/openjdk/skara/storage/HostedRepositoryStorage.java @@ -57,7 +57,7 @@ class HostedRepositoryStorage implements Storage { try { Repository localRepository; try { - localRepository = Repository.materialize(localStorage, repository.url(), ref); + localRepository = Repository.materialize(localStorage, repository.url(), "+" + ref + ":storage"); } catch (IOException e) { // The remote ref may not yet exist localRepository = Repository.init(localStorage, repository.repositoryType()); diff --git a/test/build.gradle b/test/build.gradle index 07fc386a8..a9efb9399 100644 --- a/test/build.gradle +++ b/test/build.gradle @@ -26,6 +26,7 @@ module { } dependencies { + implementation project(':ci') implementation project(':json') implementation project(':census') implementation project(':vcs') diff --git a/test/src/main/java/org/openjdk/skara/test/CensusBuilder.java b/test/src/main/java/org/openjdk/skara/test/CensusBuilder.java index 6d4196962..7780d2ae7 100644 --- a/test/src/main/java/org/openjdk/skara/test/CensusBuilder.java +++ b/test/src/main/java/org/openjdk/skara/test/CensusBuilder.java @@ -208,7 +208,7 @@ private void generateVersion(Path folder) throws IOException { public HostedRepository build() { try { var host = TestHost.createNew(List.of(new HostUser(1, "cu", "Census User"))); - var repository = host.repository("census"); + var repository = host.repository("census").orElseThrow(); var folder = Files.createTempDirectory("censusbuilder"); var localRepository = Repository.init(folder, VCS.GIT); diff --git a/test/src/main/java/org/openjdk/skara/test/HostCredentials.java b/test/src/main/java/org/openjdk/skara/test/HostCredentials.java index 124186395..29de16fe7 100644 --- a/test/src/main/java/org/openjdk/skara/test/HostCredentials.java +++ b/test/src/main/java/org/openjdk/skara/test/HostCredentials.java @@ -25,8 +25,8 @@ import org.openjdk.skara.forge.*; import org.openjdk.skara.host.*; import org.openjdk.skara.issuetracker.*; -import org.openjdk.skara.network.URIBuilder; import org.openjdk.skara.json.*; +import org.openjdk.skara.network.URIBuilder; import org.openjdk.skara.proxy.HttpProxy; import org.openjdk.skara.vcs.*; @@ -44,6 +44,7 @@ public class HostCredentials implements AutoCloseable { private final String testName; private final Credentials credentials; private final List pullRequestsToBeClosed = new ArrayList<>(); + private final List issuesToBeClosed = new ArrayList<>(); private HostedRepository credentialsLock; private int nextHostIndex; @@ -72,12 +73,15 @@ public Forge createRepositoryHost(int userIndex) { var hostUri = URIBuilder.base(config.get("host").asString()).build(); var apps = config.get("apps").asArray(); var key = configDir.resolve(apps.get(userIndex).get("key").asString()); - return ForgeFactory.createGitHubHost(hostUri, - null, - null, - key.toString(), - apps.get(userIndex).get("id").asString(), - apps.get(userIndex).get("installation").asString()); + try { + var keyContents = Files.readString(key, StandardCharsets.UTF_8); + var pat = new Credential(apps.get(userIndex).get("id").asString() + ";" + + apps.get(userIndex).get("installation").asString(), + keyContents); + return Forge.from("github", hostUri, pat, null); + } catch (IOException e) { + throw new RuntimeException("Cannot read private key: " + key); + } } @Override @@ -87,7 +91,7 @@ public IssueTracker createIssueHost(int userIndex) { @Override public HostedRepository getHostedRepository(Forge host) { - return host.repository(config.get("project").asString()); + return host.repository(config.get("project").asString()).orElseThrow(); } @Override @@ -112,9 +116,9 @@ private static class GitLabCredentials implements Credentials { public Forge createRepositoryHost(int userIndex) { var hostUri = URIBuilder.base(config.get("host").asString()).build(); var users = config.get("users").asArray(); - var pat = new PersonalAccessToken(users.get(userIndex).get("name").asString(), + var pat = new Credential(users.get(userIndex).get("name").asString(), users.get(userIndex).get("pat").asString()); - return ForgeFactory.createGitLabHost(hostUri, pat); + return Forge.from("gitlab", hostUri, pat, null); } @Override @@ -124,7 +128,44 @@ public IssueTracker createIssueHost(int userIndex) { @Override public HostedRepository getHostedRepository(Forge host) { - return host.repository(config.get("project").asString()); + return host.repository(config.get("project").asString()).orElseThrow(); + } + + @Override + public IssueProject getIssueProject(IssueTracker host) { + return host.project(config.get("project").asString()); + } + + @Override + public String getNamespaceName() { + return config.get("namespace").asString(); + } + } + + private static class JiraCredentials implements Credentials { + private final JSONObject config; + + JiraCredentials(JSONObject config) { + this.config = config; + } + + @Override + public Forge createRepositoryHost(int userIndex) { + throw new RuntimeException("not supported"); + } + + @Override + public IssueTracker createIssueHost(int userIndex) { + var hostUri = URIBuilder.base(config.get("host").asString()).build(); + var users = config.get("users").asArray(); + var pat = new Credential(users.get(userIndex).get("name").asString(), + users.get(userIndex).get("pat").asString()); + return IssueTracker.from("jira", hostUri, pat, null); + } + + @Override + public HostedRepository getHostedRepository(Forge host) { + return host.repository(config.get("project").asString()).orElseThrow(); } @Override @@ -168,7 +209,7 @@ public IssueTracker createIssueHost(int userIndex) { @Override public HostedRepository getHostedRepository(Forge host) { - return host.repository("test"); + return host.repository("test").orElseThrow(); } @Override @@ -197,6 +238,8 @@ private Credentials parseEntry(JSONObject entry, Path credentialsPath) { return new GitLabCredentials(entry); case "github": return new GitHubCredentials(entry, credentialsPath); + case "jira": + return new JiraCredentials(entry); default: throw new RuntimeException("Unknown entry type: " + entry.get("type").asString()); } @@ -327,6 +370,12 @@ public PullRequest createPullRequest(HostedRepository hostedRepository, String t return createPullRequest(hostedRepository, targetRef, sourceRef, title, false); } + public Issue createIssue(IssueProject issueProject, String title) { + var issue = issueProject.createIssue(title, List.of()); + issuesToBeClosed.add(issue); + return issue; + } + public CensusBuilder getCensusBuilder() { return CensusBuilder.create(credentials.getNamespaceName()); } @@ -336,6 +385,9 @@ public void close() { for (var pr : pullRequestsToBeClosed) { pr.setState(PullRequest.State.CLOSED); } + for (var issue : issuesToBeClosed) { + issue.setState(Issue.State.CLOSED); + } if (credentialsLock != null) { try { releaseLock(credentialsLock); diff --git a/test/src/main/java/org/openjdk/skara/test/IssueData.java b/test/src/main/java/org/openjdk/skara/test/IssueData.java index 02ea84e82..860753edd 100644 --- a/test/src/main/java/org/openjdk/skara/test/IssueData.java +++ b/test/src/main/java/org/openjdk/skara/test/IssueData.java @@ -22,6 +22,7 @@ */ package org.openjdk.skara.test; +import org.openjdk.skara.host.HostUser; import org.openjdk.skara.issuetracker.*; import java.time.ZonedDateTime; @@ -33,6 +34,7 @@ class IssueData { String title = ""; final List comments = new ArrayList<>(); final Set labels = new HashSet<>(); + final List assignees = new ArrayList<>(); ZonedDateTime created = ZonedDateTime.now(); ZonedDateTime lastUpdate = created; } diff --git a/test/src/main/java/org/openjdk/skara/test/SMTPServer.java b/test/src/main/java/org/openjdk/skara/test/SMTPServer.java index bc41e59b6..eed3a74fd 100644 --- a/test/src/main/java/org/openjdk/skara/test/SMTPServer.java +++ b/test/src/main/java/org/openjdk/skara/test/SMTPServer.java @@ -71,6 +71,9 @@ private void handleSession(SMTPSession session) throws IOException { inHeader = false; } } + if (line.startsWith(".")) { + line = line.substring(1); + } mailBody.append(line); mailBody.append("\n"); } diff --git a/test/src/main/java/org/openjdk/skara/test/TestHost.java b/test/src/main/java/org/openjdk/skara/test/TestHost.java index 6da131698..28684229f 100644 --- a/test/src/main/java/org/openjdk/skara/test/TestHost.java +++ b/test/src/main/java/org/openjdk/skara/test/TestHost.java @@ -78,7 +78,7 @@ public boolean isValid() { } @Override - public HostedRepository repository(String name) { + public Optional repository(String name) { Repository localRepository; if (data.repositories.containsKey(name)) { localRepository = data.repositories.get(name); @@ -89,7 +89,7 @@ public HostedRepository repository(String name) { localRepository = createLocalRepository(); data.repositories.put(name, localRepository); } - return new TestHostedRepository(this, name, localRepository); + return Optional.of(new TestHostedRepository(this, name, localRepository)); } @Override diff --git a/test/src/main/java/org/openjdk/skara/test/TestIssue.java b/test/src/main/java/org/openjdk/skara/test/TestIssue.java index fc9d758b8..5ec0f3e33 100644 --- a/test/src/main/java/org/openjdk/skara/test/TestIssue.java +++ b/test/src/main/java/org/openjdk/skara/test/TestIssue.java @@ -170,11 +170,13 @@ public URI webUrl() { @Override public List assignees() { - throw new RuntimeException("not implemented yet"); + return new ArrayList<>(data.assignees); } @Override public void setAssignees(List assignees) { - throw new RuntimeException("not implemented yet"); + data.assignees.clear(); + data.assignees.addAll(assignees); + data.lastUpdate = ZonedDateTime.now(); } } diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/ReadOnlyRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/ReadOnlyRepository.java index 6f79770fa..6634e49ae 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/ReadOnlyRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/ReadOnlyRepository.java @@ -77,7 +77,9 @@ default List files(Hash h, Path... paths) throws IOException { void dump(FileEntry entry, Path to) throws IOException; List status(Hash from, Hash to) throws IOException; Diff diff(Hash base, Hash head) throws IOException; + Diff diff(Hash base, Hash head, List files) throws IOException; Diff diff(Hash head) throws IOException; + Diff diff(Hash head, List files) throws IOException; List config(String key) throws IOException; Repository copyTo(Path destination) throws IOException; String pullPath(String remote) throws IOException; @@ -86,6 +88,7 @@ default List files(Hash h, Path... paths) throws IOException { Optional upstreamFor(Branch branch) throws IOException; List remoteBranches(String remote) throws IOException; List remotes() throws IOException; + List submodules() throws IOException; static Optional get(Path p) throws IOException { return Repository.get(p).map(r -> r); @@ -94,4 +97,6 @@ static Optional get(Path p) throws IOException { static boolean exists(Path p) throws IOException { return Repository.exists(p); } + + Optional annotate(Tag tag) throws IOException; } diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/Repository.java b/vcs/src/main/java/org/openjdk/skara/vcs/Repository.java index c905080af..063bc4f6b 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/Repository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/Repository.java @@ -107,6 +107,7 @@ Hash amend(String message, default void setPaths(String remote, String pullPath) throws IOException { setPaths(remote, pullPath, null); } + void addSubmodule(String pullPath, Path path) throws IOException; default void push(Hash hash, URI uri, String ref) throws IOException { push(hash, uri, ref, false); diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/Submodule.java b/vcs/src/main/java/org/openjdk/skara/vcs/Submodule.java new file mode 100644 index 000000000..102475180 --- /dev/null +++ b/vcs/src/main/java/org/openjdk/skara/vcs/Submodule.java @@ -0,0 +1,54 @@ +package org.openjdk.skara.vcs; + +import java.nio.file.Path; +import java.util.Objects; + +public class Submodule { + private final Hash hash; + private final Path path; + private final String pullPath; + + public Submodule(Hash hash, Path path, String pullPath) { + this.hash = hash; + this.path = path; + this.pullPath = pullPath; + } + + public Hash hash() { + return hash; + } + + public Path path() { + return path; + } + + public String pullPath() { + return pullPath; + } + + @Override + public String toString() { + return pullPath + " " + hash + " " + path; + } + + @Override + public int hashCode() { + return Objects.hash(hash, path, pullPath); + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + + if (!(other instanceof Submodule)) { + return false; + } + + var o = (Submodule) other; + return Objects.equals(hash, o.hash) && + Objects.equals(path, o.path) && + Objects.equals(pullPath, o.pullPath); + } +} diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/Tag.java b/vcs/src/main/java/org/openjdk/skara/vcs/Tag.java index 61b4ad785..7c8d16ce3 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/Tag.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/Tag.java @@ -24,8 +24,73 @@ import java.util.Objects; import java.util.Optional; +import java.time.ZonedDateTime; public class Tag { + public static class Annotated { + private final String name; + private final Hash target; + private final Author author; + private final ZonedDateTime date; + private final String message; + + public Annotated(String name, Hash target, Author author, ZonedDateTime date, String message) { + this.name = name; + this.target = target; + this.author = author; + this.date = date; + this.message = message; + } + + public String name() { + return name; + } + + public Hash target() { + return target; + } + + public Author author() { + return author; + } + + public ZonedDateTime date() { + return date; + } + + public String message() { + return message; + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } + + if (!(other instanceof Annotated)) { + return false; + } + + var o = (Annotated) other; + return Objects.equals(name, o.name) && + Objects.equals(target, o.target) && + Objects.equals(author, o.author) && + Objects.equals(date, o.date) && + Objects.equals(message, o.message); + } + + @Override + public int hashCode() { + return Objects.hash(name, target, author, date, message); + } + + @Override + public String toString() { + return name + " -> " + target.hex(); + } + } + private final String name; public Tag(String name) { diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java index 245a21683..7c433d03a 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/git/GitRepository.java @@ -311,7 +311,7 @@ public void reset(Hash target, boolean hard) throws IOException { @Override public void revert(Hash h) throws IOException { - try (var p = capture("git", "checkout", h.hex(), "--", ".")) { + try (var p = capture("git", "checkout", "--recurse-submodules", h.hex(), "--", ".")) { await(p); } } @@ -330,7 +330,7 @@ public Repository reinitialize() throws IOException { @Override public Hash fetch(URI uri, String refspec) throws IOException { - try (var p = capture("git", "fetch", "--tags", uri.toString(), refspec)) { + try (var p = capture("git", "fetch", "--recurse-submodules=on-demand", "--tags", uri.toString(), refspec)) { await(p); return resolve("FETCH_HEAD").get(); } @@ -338,14 +338,14 @@ public Hash fetch(URI uri, String refspec) throws IOException { @Override public void fetchAll() throws IOException { - try (var p = capture("git", "fetch", "--tags", "--prune", "--prune-tags", "--all")) { + try (var p = capture("git", "fetch", "--recurse-submodules=on-demand", "--tags", "--prune", "--prune-tags", "--all")) { await(p); } } private void checkout(String ref, boolean force) throws IOException { var cmd = new ArrayList(); - cmd.addAll(List.of("git", "-c", "advice.detachedHead=false", "checkout")); + cmd.addAll(List.of("git", "-c", "advice.detachedHead=false", "checkout", "--recurse-submodules")); if (force) { cmd.add("--force"); } @@ -831,11 +831,21 @@ public List status(Hash from, Hash to) throws IOException { @Override public Diff diff(Hash from) throws IOException { - return diff(from, null); + return diff(from, List.of()); + } + + @Override + public Diff diff(Hash from, List files) throws IOException { + return diff(from, null, files); } @Override public Diff diff(Hash from, Hash to) throws IOException { + return diff(from, to, List.of()); + } + + @Override + public Diff diff(Hash from, Hash to, List files) throws IOException { var cmd = new ArrayList<>(List.of("git", "diff", "--patch", "--find-renames=99%", "--find-copies=99%", @@ -850,6 +860,13 @@ public Diff diff(Hash from, Hash to) throws IOException { cmd.add(to.hex()); } + if (files != null && !files.isEmpty()) { + cmd.add("--"); + for (var file : files) { + cmd.add(file.toString()); + } + } + var p = start(cmd); try { var patches = UnifiedDiffParser.parseGitRaw(p.getInputStream()); @@ -885,7 +902,7 @@ public static Optional get(Path p) throws IOException { @Override public Repository copyTo(Path destination) throws IOException { - try (var p = capture("git", "clone", root().toString(), destination.toString())) { + try (var p = capture("git", "clone", "--recurse-submodules", root().toString(), destination.toString())) { await(p); } @@ -1021,6 +1038,8 @@ public static Repository clone(URI from, Path to, boolean isBare) throws IOExcep cmd.addAll(List.of("git", "clone")); if (isBare) { cmd.add("--bare"); + } else { + cmd.add("--recurse-submodules"); } cmd.addAll(List.of(from.toString(), to.toString())); try (var p = capture(Path.of("").toAbsolutePath(), cmd)) { @@ -1101,4 +1120,84 @@ public List remotes() throws IOException { } return remotes; } + + @Override + public void addSubmodule(String pullPath, Path path) throws IOException { + try (var p = capture("git", "submodule", "add", pullPath, path.toString())) { + await(p); + } + } + + @Override + public List submodules() throws IOException { + var gitModules = root().resolve(".gitmodules"); + if (!Files.exists(gitModules)) { + return List.of(); + } + + var urls = new HashMap(); + var paths = new HashMap(); + try (var p = capture("git", "config", "--file", gitModules.toAbsolutePath().toString(), + "--list")) { + for (var line : await(p).stdout()) { + if (line.startsWith("submodule.")) { + line = line.substring("submodule.".length()); + var parts = line.split("="); + var nameAndProperty = parts[0].split("\\."); + var name = nameAndProperty[0]; + var prop = nameAndProperty[1]; + var value = parts[1]; + if (prop.equals("path")) { + paths.put(name, value); + } else if (prop.equals("url")) { + urls.put(name, value); + } else { + throw new IOException("Unexpected submodule property: " + prop); + } + } + } + } + + var hashes = new HashMap(); + try (var p = capture("git", "submodule", "status")) { + for (var line : await(p).stdout()) { + var parts = line.substring(1).split(" "); + var hash = parts[0]; + var path = parts[1]; + hashes.put(path, hash); + } + } + + var modules = new ArrayList(); + for (var name : paths.keySet()) { + var url = urls.get(name); + var path = paths.get(name); + var hash = hashes.get(path); + + modules.add(new Submodule(new Hash(hash), Path.of(path), url)); + } + + return modules; + } + + @Override + public Optional annotate(Tag tag) throws IOException { + var ref = "refs/tags/" + tag.name(); + var format = "%(refname:short)%0a%(*objectname)%0a%(taggername) %(taggeremail)%0a%(taggerdate:iso-strict)%0a%(contents)"; + try (var p = capture("git", "for-each-ref", "--format", format, ref)) { + var lines = await(p).stdout(); + if (lines.size() >= 4) { + var name = lines.get(0); + var target = new Hash(lines.get(1)); + var author = Author.fromString(lines.get(2)); + + var formatter = DateTimeFormatter.ISO_OFFSET_DATE_TIME; + var date = ZonedDateTime.parse(lines.get(3), formatter); + var message = String.join("\n", lines.subList(4, lines.size())); + + return Optional.of(new Tag.Annotated(name, target, author, date, message)); + } + return Optional.empty(); + } + } } diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java b/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java index 959921764..8301663c6 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/hg/HgRepository.java @@ -741,11 +741,21 @@ public void revert(Hash parent) throws IOException { @Override public Diff diff(Hash from) throws IOException { - return diff(from, null); + return diff(from, List.of()); + } + + @Override + public Diff diff(Hash from, List files) throws IOException { + return diff(from, null, files); } @Override public Diff diff(Hash from, Hash to) throws IOException { + return diff(from, to, List.of()); + } + + @Override + public Diff diff(Hash from, Hash to, List files) throws IOException { var ext = Files.createTempFile("ext", ".py"); copyResource(EXT_PY, ext); @@ -755,6 +765,11 @@ public Diff diff(Hash from, Hash to) throws IOException { cmd.add(to.hex()); } + if (files != null) { + var filenames = files.stream().map(Path::toString).collect(Collectors.toList()); + cmd.add("--files=" + String.join(",", filenames)); + } + var p = start(cmd); try { var patches = UnifiedDiffParser.parseGitRaw(p.getInputStream()); @@ -1131,4 +1146,73 @@ public List remotes() throws IOException { } return remotes; } + + @Override + public void addSubmodule(String pullPath, Path path) throws IOException { + var uri = Files.exists(Path.of(pullPath)) ? Path.of(pullPath).toUri().toString() : pullPath; + HgRepository.clone(URI.create(uri), root().resolve(path).toAbsolutePath(), false); + var hgSub = root().resolve(".hgsub"); + Files.writeString(hgSub, path.toString() + " = " + pullPath + "\n", + StandardOpenOption.WRITE, StandardOpenOption.APPEND, StandardOpenOption.CREATE); + add(List.of(hgSub)); + } + + @Override + public List submodules() throws IOException { + var hgSub = root().resolve(".hgsub"); + var hgSubState = root().resolve(".hgsubstate"); + if (!(Files.exists(hgSub) && Files.exists(hgSubState))) { + return List.of(); + } + + var urls = new HashMap(); + for (var line : Files.readAllLines(hgSub)) { + var parts = line.split("="); + var path = parts[0].trim(); + var url = parts[1].trim(); + urls.put(path, url); + } + + var hashes = new HashMap(); + for (var line : Files.readAllLines(hgSubState)) { + var parts = line.split(" "); + var hash = parts[0]; + var path = parts[1]; + hashes.put(path, hash); + } + + var modules = new ArrayList(); + for (var path : urls.keySet()) { + var url = urls.get(path); + var hash = hashes.get(path); + modules.add(new Submodule(new Hash(hash), Path.of(path), url)); + } + + return modules; + } + + @Override + public Optional annotate(Tag tag) throws IOException { + var hgtags = root().resolve(".hgtags"); + if (!Files.exists(hgtags)) { + return Optional.empty(); + } + try (var p = capture("hg", "annotate", hgtags.toString())) { + var reversed = new ArrayList<>(await(p).stdout()); + Collections.reverse(reversed); + for (var line : reversed) { + var parts = line.split(" "); + var tagName = parts[2]; + if (tagName.equals(tag.name())) { + var target = new Hash(parts[1]); + var rev = parts[0].substring(0, parts[0].length() - 1).trim(); // skip last ':' and ev. whitespace + var hash = resolve(rev).orElseThrow(IOException::new); + var commit = lookup(hash).orElseThrow(IOException::new); + var message = String.join("\n", commit.message()) + "\n"; + return Optional.of(new Tag.Annotated(tagName, target, commit.author(), commit.date(), message)); + } + } + } + return Optional.empty(); + } } diff --git a/vcs/src/main/java/org/openjdk/skara/vcs/openjdk/OpenJDKTag.java b/vcs/src/main/java/org/openjdk/skara/vcs/openjdk/OpenJDKTag.java index 37df3e33d..6969123dd 100644 --- a/vcs/src/main/java/org/openjdk/skara/vcs/openjdk/OpenJDKTag.java +++ b/vcs/src/main/java/org/openjdk/skara/vcs/openjdk/OpenJDKTag.java @@ -52,16 +52,19 @@ private OpenJDKTag(Tag tag, String prefix, String version, String buildPrefix, S * jdk7u40-b20 -> jdk7u40 7u40 u20 -b 29 * hs24-b30 -> hs24 24 -b 30 * hs23.6-b19 -> hs23.6 23.6 .6 -b 19 + * 11.1+22 -> 11.1 11.1 .1 + 22 */ private final static String legacyOpenJDKVersionPattern = "(jdk([0-9]{1,2}(u[0-9]{1,3})?))"; private final static String legacyHSVersionPattern = "((hs[0-9]{1,2}(\\.[0-9]{1,3})?))"; private final static String legacyBuildPattern = "(-b)([0-9]{2,3})"; private final static String OpenJDKVersionPattern = "(jdk-([0-9]+(\\.[0-9]){0,3}))(\\+)([0-9]+)"; + private final static String OpenJFXVersionPattern = "((?:jdk-){0,1}([1-9](?:(?:[0-9]*)(\\.(?:0|[1-9][0-9]*)){0,3})))(?:(\\+)([0-9]+)|(-ga))"; private final static List tagPatterns = List.of(Pattern.compile(legacyOpenJDKVersionPattern + legacyBuildPattern), Pattern.compile(legacyHSVersionPattern + legacyBuildPattern), - Pattern.compile(OpenJDKVersionPattern)); + Pattern.compile(OpenJDKVersionPattern), + Pattern.compile(OpenJFXVersionPattern)); /** * Attempts to create an OpenJDKTag instance from a general Tag. @@ -106,6 +109,9 @@ public String version() { * @return */ public int buildNum() { + if (buildNum == null) { + return 0; + } return Integer.parseInt(buildNum); } diff --git a/vcs/src/main/resources/ext.py b/vcs/src/main/resources/ext.py index aaff800c4..2d6d90b0c 100644 --- a/vcs/src/main/resources/ext.py +++ b/vcs/src/main/resources/ext.py @@ -29,13 +29,13 @@ import sys # space separated version list -testedwith = '4.9.2 5.0.2' +testedwith = '4.9.2 5.0.2 5.2.1' def mode(fctx): flags = fctx.flags() - if flags == '': return '100644' - if flags == 'x': return '100755' - if flags == 'l': return '120000' + if flags == b'': return b'100644' + if flags == b'x': return b'100755' + if flags == b'l': return b'120000' def ratio(a, b, threshold): s = difflib.SequenceMatcher(None, a, b) @@ -48,27 +48,30 @@ def ratio(a, b, threshold): return 0 return ratio -def encode(s): - return s.decode('utf-8').encode('utf-8') - def write(s): - sys.stdout.write(encode(s)) + if sys.version_info >= (3, 0): + sys.stdout.buffer.write(s) + else: + sys.stdout.write(s) def writeln(s): write(s) - sys.stdout.write(encode('\n')) + write(b'\n') + +def int_to_str(i): + return str(i).encode('ascii') def _match_exact(root, cwd, files, badfn=None): """ Wrapper for mercurial.match.exact that ignores some arguments based on the used version """ - if mercurial.util.version().startswith("5"): + if mercurial.util.version().startswith(b"5"): return mercurial.match.exact(files, badfn) else: return mercurial.match.exact(root, cwd, files, badfn) def _diff_git_raw(repo, ctx1, ctx2, modified, added, removed, showPatch): - nullHash = '0' * 40 + nullHash = b'0' * 40 removed_copy = set(removed) for path in added: @@ -82,33 +85,34 @@ def _diff_git_raw(repo, ctx1, ctx2, modified, added, removed, showPatch): for path in sorted(modified | added | removed_copy): if path in modified: fctx = ctx2.filectx(path) - writeln(':{} {} {} {} M\t{}'.format(mode(ctx1.filectx(path)), mode(fctx), nullHash, nullHash, fctx.path())) + writeln(b':' + mode(ctx1.filectx(path)) + b' ' + mode(fctx) + b' ' + nullHash + b' ' + nullHash + b' M\t' + fctx.path()) elif path in added: fctx = ctx2.filectx(path) if not fctx.renamed(): - writeln(':000000 {} {} {} A\t{}'.format(mode(fctx), nullHash, nullHash, fctx.path())) + writeln(b':000000 ' + mode(fctx) + b' ' + nullHash + b' ' + nullHash + b' A\t' + fctx.path()) else: parent = fctx.p1() - score = int(ratio(parent.data(), fctx.data(), 0.5) * 100) + score = int_to_str(int(ratio(parent.data(), fctx.data(), 0.5) * 100)) old_path, _ = fctx.renamed() if old_path in removed: - operation = 'R' + operation = b'R' else: - operation = 'C' + operation = b'C' - writeln(':{} {} {} {} {}{}\t{}\t{}'.format(mode(parent), mode(fctx), nullHash, nullHash, operation, score, old_path, path)) + write(b':' + mode(parent) + b' ' + mode(fctx) + b' ' + nullHash + b' ' + nullHash + b' ') + writeln(operation + score + b'\t' + old_path + b'\t' + path) elif path in removed_copy: fctx = ctx1.filectx(path) - writeln(':{} 000000 {} {} D\t{}'.format(mode(fctx), nullHash, nullHash, path)) + writeln(b':' + mode(fctx) + b' 000000 ' + nullHash + b' ' + nullHash + b' D\t' + path) if showPatch: - writeln('') + writeln(b'') match = _match_exact(repo.root, repo.getcwd(), list(modified) + list(added) + list(removed_copy)) opts = mercurial.mdiff.diffopts(git=True, nodates=True, context=0, showfunc=True) for d in mercurial.patch.diff(repo, ctx1.node(), ctx2.node(), match=match, opts=opts): - sys.stdout.write(d) + write(d) def really_differs(repo, p1, p2, ctx, files): # workaround bug in hg (present since forever): @@ -151,8 +155,8 @@ def decorator(func): revsingle = mercurial.cmdutil.revsingle revrange = mercurial.cmdutil.revrange -@command('diff-git-raw', [('', 'patch', False, '')], 'hg diff-git-raw rev1 [rev2]') -def diff_git_raw(ui, repo, rev1, rev2=None, **opts): +@command(b'diff-git-raw', [(b'', b'patch', False, b''), (b'', b'files', b'', b'')], b'hg diff-git-raw rev1 [rev2]') +def diff_git_raw(ui, repo, rev1, rev2=None, *files, **opts): ctx1 = revsingle(repo, rev1) if rev2 != None: @@ -163,18 +167,26 @@ def diff_git_raw(ui, repo, rev1, rev2=None, **opts): status = repo.status(ctx1) modified, added, removed = [set(l) for l in status[:3]] + + files = opts['files'] + if files != b'': + wanted = set(files.split(b',')) + modified = modified & wanted + added = added & wanted + removed = removed & wanted + _diff_git_raw(repo, ctx1, ctx2, modified, added, removed, opts['patch']) -@command('log-git', [('', 'reverse', False, ''), ('l', 'limit', -1, '')], 'hg log-git ') +@command(b'log-git', [(b'', b'reverse', False, b''), (b'l', b'limit', -1, b'')], b'hg log-git ') def log_git(ui, repo, revs=None, **opts): if len(repo) == 0: return if revs == None: if opts['reverse']: - revs = '0:tip' + revs = b'0:tip' else: - revs = 'tip:0' + revs = b'tip:0' limit = opts['limit'] i = 0 @@ -209,7 +221,7 @@ def log_git(ui, repo, revs=None, **opts): combined_added_p2 = really_differs(repo, p1, p2, ctx, combined_added_p2) _diff_git_raw(repo, p1, ctx, combined_modified_p1, combined_added_p1, removed_both, True) - writeln('#@!_-=&') + writeln(b'#@!_-=&') _diff_git_raw(repo, p2, ctx, combined_modified_p2, combined_added_p2, removed_both, True) i += 1 @@ -217,44 +229,44 @@ def log_git(ui, repo, revs=None, **opts): break def __dump_metadata(ctx): - writeln('#@!_-=&') + writeln(b'#@!_-=&') writeln(ctx.hex()) - writeln(str(ctx.rev())) + writeln(int_to_str(ctx.rev())) writeln(ctx.branch()) parents = ctx.parents() - writeln(' '.join([str(p.hex()) for p in parents])) - writeln(' '.join([str(p.rev()) for p in parents])) + writeln(b' '.join([p.hex() for p in parents])) + writeln(b' '.join([int_to_str(p.rev()) for p in parents])) writeln(ctx.user()) - date = datestr(ctx.date(), format='%Y-%m-%d %H:%M:%S%z') + date = datestr(ctx.date(), format=b'%Y-%m-%d %H:%M:%S%z') writeln(date) - description = encode(ctx.description()) - writeln(str(len(description))) + description = ctx.description() + writeln(int_to_str(len(description))) write(description) def __dump(repo, start, end): - for rev in xrange(start, end): + for rev in range(start, end): ctx = revsingle(repo, rev) __dump_metadata(ctx) parents = ctx.parents() modified, added, removed = repo.status(parents[0], ctx)[:3] - writeln(str(len(modified))) - writeln(str(len(added))) - writeln(str(len(removed))) + writeln(int_to_str(len(modified))) + writeln(int_to_str(len(added))) + writeln(int_to_str(len(removed))) for filename in added + modified: fctx = ctx.filectx(filename) writeln(filename) - writeln(' '.join(fctx.flags())) + writeln(b' '.join(fctx.flags())) content = fctx.data() - writeln(str(len(content))) - sys.stdout.write(content) + writeln(int_to_str(len(content))) + write(content) for filename in removed: writeln(filename) @@ -264,38 +276,38 @@ def pretxnclose(ui, repo, **kwargs): end = revsingle(repo, kwargs['node_last']) __dump(repo, start.rev(), end.rev() + 1) -@command('dump', [], 'hg dump') +@command(b'dump', [], b'hg dump') def dump(ui, repo, **opts): __dump(repo, 0, len(repo)) -@command('metadata', [], 'hg metadata') +@command(b'metadata', [], b'hg metadata') def dump(ui, repo, revs=None, **opts): if revs == None: - revs = "0:tip" + revs = b"0:tip" for r in revrange(repo, [revs]): ctx = repo[r] __dump_metadata(ctx) -@command('ls-tree', [], 'hg ls-tree') +@command(b'ls-tree', [], b'hg ls-tree') def ls_tree(ui, repo, rev, **opts): - nullHash = '0' * 40 + nullHash = b'0' * 40 ctx = revsingle(repo, rev) for filename in ctx.manifest(): fctx = ctx.filectx(filename) - if 'x' in fctx.flags(): - write('100755 blob ') + if b'x' in fctx.flags(): + write(b'100755 blob ') else: - write('100644 blob ') + write(b'100644 blob ') write(nullHash) - write('\t') + write(b'\t') writeln(filename) -@command('ls-remote', [], 'hg ls-remote PATH') +@command(b'ls-remote', [], b'hg ls-remote PATH') def ls_remote(ui, repo, path, **opts): peer = mercurial.hg.peer(ui or repo, opts, ui.expandpath(path)) for branch, heads in peer.branchmap().iteritems(): for head in heads: write(mercurial.node.hex(head)) - write("\t") + write(b"\t") writeln(branch) diff --git a/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java b/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java index db105fecd..a0bc7ec15 100644 --- a/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java +++ b/vcs/src/test/java/org/openjdk/skara/vcs/RepositoryTests.java @@ -33,6 +33,7 @@ import java.net.URI; import java.nio.file.*; import java.nio.file.attribute.*; +import java.time.ZonedDateTime; import java.util.*; import java.util.stream.Collectors; @@ -1848,4 +1849,149 @@ void testRemoteBranches(VCS vcs) throws IOException { assertEquals(upstream.defaultBranch().name(), ref.name()); } } + + @ParameterizedTest + @EnumSource(VCS.class) + void testSubmodulesOnEmptyRepo(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory()) { + var repo = Repository.init(dir.path(), vcs); + assertEquals(List.of(), repo.submodules()); + } + } + + @ParameterizedTest + @EnumSource(VCS.class) + void testSubmodulesOnRepoWithNoSubmodules(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory()) { + var repo = Repository.init(dir.path().resolve("repo"), vcs); + var readme = repo.root().resolve("README"); + Files.writeString(readme, "Hello\n"); + repo.add(readme); + repo.commit("Added README", "duke", "duke@openjdk.org"); + assertEquals(List.of(), repo.submodules()); + } + } + + @ParameterizedTest + @EnumSource(VCS.class) + void testSubmodulesOnRepoWithSubmodule(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory()) { + var submodule = Repository.init(dir.path().resolve("submodule"), vcs); + var readme = submodule.root().resolve("README"); + Files.writeString(readme, "Hello\n"); + submodule.add(readme); + var head = submodule.commit("Added README", "duke", "duke@openjdk.org"); + + var repo = Repository.init(dir.path().resolve("repo"), vcs); + var pullPath = submodule.root().toAbsolutePath().toString(); + repo.addSubmodule(pullPath, Path.of("sub")); + repo.commit("Added submodule", "duke", "duke@openjdk.org"); + + var submodules = repo.submodules(); + assertEquals(1, submodules.size()); + var module = submodules.get(0); + assertEquals(Path.of("sub"), module.path()); + assertEquals(head, module.hash()); + assertEquals(pullPath, module.pullPath()); + } + } + + @ParameterizedTest + @EnumSource(VCS.class) + void testAnnotateTag(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory()) { + var repo = Repository.init(dir.path(), vcs); + var readme = repo.root().resolve("README"); + var now = ZonedDateTime.now(); + Files.writeString(readme, "Hello\n"); + repo.add(readme); + var head = repo.commit("Added README", "duke", "duke@openjdk.org"); + var tag = repo.tag(head, "1.0", "Added tag 1.0 for HEAD\n", "duke", "duke@openjdk.org"); + var annotated = repo.annotate(tag).get(); + + assertEquals("1.0", annotated.name()); + assertEquals(head, annotated.target()); + assertEquals(new Author("duke", "duke@openjdk.org"), annotated.author()); + assertEquals(now.getYear(), annotated.date().getYear()); + assertEquals(now.getMonth(), annotated.date().getMonth()); + assertEquals(now.getDayOfYear(), annotated.date().getDayOfYear()); + assertEquals(now.getHour(), annotated.date().getHour()); + assertEquals(now.getOffset(), annotated.date().getOffset()); + assertEquals("Added tag 1.0 for HEAD\n", annotated.message()); + } + } + + @ParameterizedTest + @EnumSource(VCS.class) + void testAnnotateTagOnMissingTag(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory()) { + var repo = Repository.init(dir.path(), vcs); + var readme = repo.root().resolve("README"); + var now = ZonedDateTime.now(); + Files.writeString(readme, "Hello\n"); + repo.add(readme); + var head = repo.commit("Added README", "duke", "duke@openjdk.org"); + + assertEquals(Optional.empty(), repo.annotate(new Tag("unknown"))); + } + } + + @ParameterizedTest + @EnumSource(VCS.class) + void testAnnotateTagOnEmptyRepo(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory()) { + var repo = Repository.init(dir.path(), vcs); + assertEquals(Optional.empty(), repo.annotate(new Tag("unknown"))); + } + } + + @ParameterizedTest + @EnumSource(VCS.class) + void testDiffWithFileList(VCS vcs) throws IOException { + try (var dir = new TemporaryDirectory(false)) { + var repo = Repository.init(dir.path(), vcs); + var readme = repo.root().resolve("README"); + Files.writeString(readme, "Hello\n"); + repo.add(readme); + + var contribute = repo.root().resolve("CONTRIBUTE"); + Files.writeString(contribute, "1. Make changes\n"); + repo.add(contribute); + + var first = repo.commit("Added README and CONTRIBUTE", "duke", "duke@openjdk.org"); + Files.writeString(readme, "World\n", WRITE, APPEND); + Files.writeString(contribute, "2. Run git commit", WRITE, APPEND); + + var diff = repo.diff(first, List.of(Path.of("README"))); + assertEquals(1, diff.added()); + assertEquals(0, diff.modified()); + assertEquals(0, diff.removed()); + var patches = diff.patches(); + assertEquals(1, patches.size()); + var patch = patches.get(0); + assertTrue(patch.isTextual()); + assertTrue(patch.status().isModified()); + assertEquals(Path.of("README"), patch.source().path().get()); + assertEquals(Path.of("README"), patch.target().path().get()); + + repo.add(readme); + repo.add(contribute); + var second = repo.commit("Updates to both README and CONTRIBUTE", "duke", "duke@openjdk.org"); + + diff = repo.diff(first, second, List.of(Path.of("CONTRIBUTE"))); + assertEquals(1, diff.added()); + assertEquals(0, diff.modified()); + assertEquals(0, diff.removed()); + patches = diff.patches(); + assertEquals(1, patches.size()); + patch = patches.get(0); + assertTrue(patch.isTextual()); + assertTrue(patch.status().isModified()); + assertEquals(Path.of("CONTRIBUTE"), patch.source().path().get()); + assertEquals(Path.of("CONTRIBUTE"), patch.target().path().get()); + + diff = repo.diff(first, second, List.of(Path.of("DOES_NOT_EXIST"))); + assertEquals(0, diff.patches().size()); + } + } } diff --git a/vcs/src/test/java/org/openjdk/skara/vcs/openjdk/OpenJDKTagTests.java b/vcs/src/test/java/org/openjdk/skara/vcs/openjdk/OpenJDKTagTests.java index 70a52dbf1..dd0ef2298 100644 --- a/vcs/src/test/java/org/openjdk/skara/vcs/openjdk/OpenJDKTagTests.java +++ b/vcs/src/test/java/org/openjdk/skara/vcs/openjdk/OpenJDKTagTests.java @@ -22,10 +22,11 @@ */ package org.openjdk.skara.vcs.openjdk; +import org.openjdk.skara.vcs.Tag; + import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; -import org.openjdk.skara.vcs.Tag; +import static org.junit.jupiter.api.Assertions.*; class OpenJDKTagTests { @Test @@ -92,4 +93,22 @@ void noPrevious() { assertEquals(0, jdkTag.buildNum()); assertFalse(jdkTag.previous().isPresent()); } + + @Test + void parseJfxTags() { + var tag = new Tag("12.1.3+14"); + var jdkTag = OpenJDKTag.create(tag).orElseThrow(); + assertEquals("12.1.3", jdkTag.version()); + assertEquals(14, jdkTag.buildNum()); + var previousTag = jdkTag.previous().orElseThrow(); + assertEquals(13, previousTag.buildNum()); + } + + @Test + void parseJfxTagsGa() { + var tag = new Tag("12.1-ga"); + var jdkTag = OpenJDKTag.create(tag).orElseThrow(); + assertEquals("12.1", jdkTag.version()); + assertEquals(0, jdkTag.buildNum()); + } } diff --git a/webrev/src/main/java/org/openjdk/skara/webrev/Webrev.java b/webrev/src/main/java/org/openjdk/skara/webrev/Webrev.java index 3222c4b55..90921fa50 100644 --- a/webrev/src/main/java/org/openjdk/skara/webrev/Webrev.java +++ b/webrev/src/main/java/org/openjdk/skara/webrev/Webrev.java @@ -61,6 +61,7 @@ public static class Builder { private String branch; private String issue; private String version; + private List files = List.of(); Builder(ReadOnlyRepository repository, Path output) { this.repository = repository; @@ -102,6 +103,11 @@ public Builder version(String version) { return this; } + public Builder files(List files) { + this.files = files; + return this; + } + public void generate(Hash tailEnd) throws IOException { generate(tailEnd, null); } @@ -114,9 +120,36 @@ public void generate(Hash tailEnd, Hash head) throws IOException { copyResource(CSS); copyResource(ICON); - var diff = head == null ? repository.diff(tailEnd) : repository.diff(tailEnd, head); + var diff = head == null ? + repository.diff(tailEnd, files) : + repository.diff(tailEnd, head, files); var patchFile = output.resolve(Path.of(title).getFileName().toString() + ".patch"); + var patches = diff.patches(); + if (files != null && !files.isEmpty()) { + // Sort the patches according to how they are listed in the `files` list. + var byTargetPath = new HashMap(); + var bySourcePath = new HashMap(); + for (var patch : patches) { + if (patch.target().path().isPresent()) { + byTargetPath.put(patch.target().path().get(), patch); + } else { + bySourcePath.put(patch.source().path().get(), patch); + } + } + + var sorted = new ArrayList(); + for (var file : files) { + if (byTargetPath.containsKey(file)) { + sorted.add(byTargetPath.get(file)); + } else if (bySourcePath.containsKey(file)) { + sorted.add(bySourcePath.get(file)); + } else { + throw new IOException("Filename not present in diff: " + file); + } + } + patches = sorted; + } var modified = new ArrayList(); for (var i = 0; i < patches.size(); i++) {