diff --git a/bots/checkout/build.gradle b/bots/checkout/build.gradle new file mode 100644 index 000000000..2a82f9eb3 --- /dev/null +++ b/bots/checkout/build.gradle @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +module { + name = 'org.openjdk.skara.bots.checkout' + test { + requires 'org.junit.jupiter.api' + requires 'org.openjdk.skara.test' + opens 'org.openjdk.skara.bots.checkout' to 'org.junit.platform.commons' + } +} + +dependencies { + implementation project(':bot') + implementation project(':ci') + implementation project(':vcs') + implementation project(':host') + implementation project(':forge') + implementation project(':issuetracker') + implementation project(':census') + implementation project(':process') + implementation project(':json') + implementation project(':network') + implementation project(':storage') + + testImplementation project(':test') +} diff --git a/bots/checkout/src/main/java/module-info.java b/bots/checkout/src/main/java/module-info.java new file mode 100644 index 000000000..7e9ae5c9a --- /dev/null +++ b/bots/checkout/src/main/java/module-info.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +module org.openjdk.skara.bots.checkout { + requires org.openjdk.skara.vcs; + requires org.openjdk.skara.host; + requires org.openjdk.skara.network; + requires org.openjdk.skara.bot; + requires org.openjdk.skara.process; + requires org.openjdk.skara.storage; + requires java.logging; + + provides org.openjdk.skara.bot.BotFactory with org.openjdk.skara.bots.checkout.CheckoutBotFactory; +} diff --git a/bots/checkout/src/main/java/org/openjdk/skara/bots/checkout/CheckoutBot.java b/bots/checkout/src/main/java/org/openjdk/skara/bots/checkout/CheckoutBot.java new file mode 100644 index 000000000..12204fe4e --- /dev/null +++ b/bots/checkout/src/main/java/org/openjdk/skara/bots/checkout/CheckoutBot.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package org.openjdk.skara.bots.checkout; + +import org.openjdk.skara.bot.*; +import org.openjdk.skara.vcs.*; +import org.openjdk.skara.vcs.openjdk.convert.*; +import org.openjdk.skara.storage.StorageBuilder; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.*; +import java.util.stream.Collectors; +import java.nio.file.*; +import java.nio.charset.StandardCharsets; +import java.net.URI; +import java.net.URLEncoder; +import java.util.logging.Logger; + +public class CheckoutBot implements Bot, WorkItem { + private static final Logger log = Logger.getLogger("org.openjdk.skara.bots");; + private final URI from; + private final Branch branch; + private final Path to; + private final Path storage; + private final StorageBuilder marksStorage; + + CheckoutBot(URI from, Branch branch, Path to, Path storage, StorageBuilder marksStorage) { + this.from = from; + this.branch = branch; + this.to = to; + this.storage = storage; + this.marksStorage = marksStorage; + } + + private static String urlEncode(Path p) { + return URLEncoder.encode(p.toString(), StandardCharsets.UTF_8); + } + + private static String urlEncode(URI uri) { + return URLEncoder.encode(uri.toString(), StandardCharsets.UTF_8); + } + + @Override + public boolean concurrentWith(WorkItem other) { + if (!(other instanceof CheckoutBot)) { + return true; + } + var o = (CheckoutBot) other; + return !(o.to.equals(to) || o.from.equals(from)); + } + + @Override + public String toString() { + return "CheckoutBot(" + from + ":" + branch.name() + ", " + to + ")"; + } + + @Override + public List getPeriodicItems() { + return List.of(this); + } + + @Override + public Collection run(Path scratch) { + try { + var fromDir = storage.resolve(urlEncode(from)); + Repository fromRepo = null; + if (!Files.exists(fromDir)) { + Files.createDirectories(fromDir); + log.info("Cloning Git repo " + from + " to " + fromDir); + fromRepo = Repository.clone(from, fromDir); + } else { + log.info("Getting existing Git repo repository from " + fromDir); + fromRepo = Repository.get(fromDir).orElseThrow(() -> + new IllegalStateException("Git repository vanished from " + fromDir)); + } + fromRepo.checkout(branch); + fromRepo.pull("origin", branch.name()); + + var repoName = Path.of(from.getPath()).getFileName().toString(); + var marksDir = scratch.resolve("checkout").resolve("marks").resolve(repoName); + Files.createDirectories(marksDir); + var marks = marksStorage.materialize(marksDir); + var converter = new GitToHgConverter(branch); + try { + if (!Files.exists(to)) { + log.info("Creating Hg repository at: " + to); + Files.createDirectories(to); + var toRepo = Repository.init(to, VCS.HG); + converter.convert(fromRepo, toRepo); + } else { + log.info("Found existing Hg repository at: " + to); + var toRepo = Repository.get(to).orElseThrow(() -> + new IllegalStateException("Repository vanished from " + to)); + var existing = new ArrayList(marks.current()); + log.info("Found " + existing.size() + " existing marks"); + Collections.sort(existing); + converter.convert(fromRepo, toRepo, existing); + } + } finally { + log.info("Storing " + converter.marks().size() + " marks"); + marks.put(converter.marks()); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return List.of(); + } +} diff --git a/bots/checkout/src/main/java/org/openjdk/skara/bots/checkout/CheckoutBotFactory.java b/bots/checkout/src/main/java/org/openjdk/skara/bots/checkout/CheckoutBotFactory.java new file mode 100644 index 000000000..00c29582a --- /dev/null +++ b/bots/checkout/src/main/java/org/openjdk/skara/bots/checkout/CheckoutBotFactory.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.checkout; + +import org.openjdk.skara.bot.*; +import org.openjdk.skara.vcs.*; +import org.openjdk.skara.vcs.openjdk.convert.Mark; + +import java.util.*; +import java.net.URI; +import java.nio.file.Path; +import java.util.logging.Logger; + +public class CheckoutBotFactory implements BotFactory { + private static final Logger log = Logger.getLogger("org.openjdk.skara.bots"); + + @Override + public String name() { + return "checkout"; + } + + @Override + public List create(BotConfiguration configuration) { + var specific = configuration.specific(); + var storage = configuration.storageFolder(); + + var marksRepo = configuration.repository(specific.get("marks").get("repo").asString()); + var marksUser = Author.fromString(specific.get("marks").get("author").asString()); + + var bots = new ArrayList(); + for (var repo : specific.get("repositories").asArray()) { + var from = repo.get("from").asString(); + var lastColon = from.lastIndexOf(":"); + var fromURI = URI.create(from.substring(0, lastColon)); + var fromBranch = new Branch(from.substring(lastColon + 1)); + var to = Path.of(repo.get("to").asString()); + + var repoName = Path.of(fromURI.getPath()).getFileName().toString(); + var markStorage = MarkStorage.create(marksRepo, marksUser, repoName); + + bots.add(new CheckoutBot(fromURI, fromBranch, to, storage, markStorage)); + } + + return bots; + } +} diff --git a/bots/checkout/src/main/java/org/openjdk/skara/bots/checkout/MarkStorage.java b/bots/checkout/src/main/java/org/openjdk/skara/bots/checkout/MarkStorage.java new file mode 100644 index 000000000..cff45d70c --- /dev/null +++ b/bots/checkout/src/main/java/org/openjdk/skara/bots/checkout/MarkStorage.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package org.openjdk.skara.bots.checkout; + +import org.openjdk.skara.forge.HostedRepository; +import org.openjdk.skara.vcs.Author; +import org.openjdk.skara.vcs.Hash; +import org.openjdk.skara.vcs.openjdk.convert.Mark; +import org.openjdk.skara.storage.StorageBuilder; + +import java.net.URI; +import java.util.*; +import java.util.stream.Collectors; + +class MarkStorage { + private static Mark deserializeMark(String s) { + var parts = s.split(" "); + if (!(parts.length == 3 || parts.length == 4)) { + throw new IllegalArgumentException("Unexpected string:" + s); + } + + var key = Integer.parseInt(parts[0]); + var hg = new Hash(parts[1]); + var git = new Hash(parts[2]); + + return parts.length == 3 ? new Mark(key, hg, git) : new Mark(key, hg, git, new Hash(parts[3])); + } + + private static String serialize(Collection added, Set existing) { + var marks = new ArrayList(); + var handled = new HashSet(); + for (var mark : added) { + marks.add(mark); + handled.add(mark.key()); + } + for (var mark : existing) { + if (!handled.contains(mark.key())) { + marks.add(mark); + } + } + Collections.sort(marks); + var sb = new StringBuilder(); + for (var mark : marks) { + sb.append(Integer.toString(mark.key())); + sb.append(" "); + sb.append(mark.hg().hex()); + sb.append(" "); + sb.append(mark.git().hex()); + if (mark.tag().isPresent()) { + sb.append(" "); + sb.append(mark.tag().get().hex()); + } + sb.append("\n"); + } + return sb.toString(); + } + + private static Set deserialize(String current) { + var res = current.lines() + .map(MarkStorage::deserializeMark) + .collect(Collectors.toSet()); + return res; + } + + static StorageBuilder create(HostedRepository repo, Author user, String name) { + return new StorageBuilder(name + ".marks.txt") + .remoteRepository(repo, "master", user.name(), user.email(), "Updated marks for " + name) + .serializer(MarkStorage::serialize) + .deserializer(MarkStorage::deserialize); + } +} diff --git a/bots/checkout/src/test/java/org/openjdk/skara/bots/checkout/CheckoutBotTests.java b/bots/checkout/src/test/java/org/openjdk/skara/bots/checkout/CheckoutBotTests.java new file mode 100644 index 000000000..919d2ed77 --- /dev/null +++ b/bots/checkout/src/test/java/org/openjdk/skara/bots/checkout/CheckoutBotTests.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.skara.bots.checkout; + +import org.openjdk.skara.forge.HostedRepository; +import org.openjdk.skara.storage.StorageBuilder; +import org.openjdk.skara.test.*; +import org.openjdk.skara.host.HostUser; +import org.openjdk.skara.vcs.Tag; +import org.openjdk.skara.vcs.*; + +import org.junit.jupiter.api.*; + +import java.io.IOException; +import java.nio.file.*; +import java.util.*; +import static java.nio.file.StandardOpenOption.*; + +import static org.junit.jupiter.api.Assertions.*; + +class CheckoutBotTests { + private static void populate(Repository r) throws IOException { + var readme = r.root().resolve("README"); + Files.write(readme, List.of("Hello, readme!")); + + r.add(readme); + r.commit("Add README", "duke", "duke@openjdk.java.net"); + + Files.write(readme, List.of("Another line"), WRITE, APPEND); + r.add(readme); + r.commit("Modify README", "duke", "duke@openjdk.java.net"); + + Files.write(readme, List.of("A final line"), WRITE, APPEND); + r.add(readme); + r.commit("Final README", "duke", "duke@openjdk.java.net"); + } + + @Test + void simpleConversion(TestInfo testInfo) throws IOException { + try (var tmp = new TemporaryDirectory()) { + var host = TestHost.createNew(List.of(new HostUser(0, "duke", "J. Duke"))); + var marksLocalDir = tmp.path().resolve("marks.git"); + Files.createDirectories(marksLocalDir); + var marksLocalRepo = Repository.init(marksLocalDir, VCS.GIT); + marksLocalRepo.config("receive", "denyCurrentBranch", "ignore"); + var marksHostedRepo = new TestHostedRepository(host, "marks", marksLocalRepo); + + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("scratch"); + var marksAuthor = new Author("duke", "duke@openjdk.org"); + var marksStorage = MarkStorage.create(marksHostedRepo, marksAuthor, "test"); + + var hgDir = tmp.path().resolve("hg"); + + var gitLocalDir = tmp.path().resolve("from.git"); + Files.createDirectories(gitLocalDir); + var gitLocalRepo = Repository.init(gitLocalDir, VCS.GIT); + populate(gitLocalRepo); + var gitHostedRepo = new TestHostedRepository(host, "from", gitLocalRepo); + + var bot = new CheckoutBot(gitHostedRepo.url(), gitLocalRepo.defaultBranch(), hgDir, storage, marksStorage); + var runner = new TestBotRunner(); + runner.runPeriodicItems(bot); + + var hgRepo = Repository.get(hgDir).orElseThrow(); + assertEquals(3, hgRepo.commitMetadata().size()); + } + } + + @Test + void update(TestInfo testInfo) throws IOException { + try (var tmp = new TemporaryDirectory()) { + var host = TestHost.createNew(List.of(new HostUser(0, "duke", "J. Duke"))); + var marksLocalDir = tmp.path().resolve("marks.git"); + Files.createDirectories(marksLocalDir); + var marksLocalRepo = Repository.init(marksLocalDir, VCS.GIT); + marksLocalRepo.config("receive", "denyCurrentBranch", "ignore"); + var marksHostedRepo = new TestHostedRepository(host, "marks", marksLocalRepo); + + var storage = tmp.path().resolve("storage"); + var scratch = tmp.path().resolve("scratch"); + var marksAuthor = new Author("duke", "duke@openjdk.org"); + var marksStorage = MarkStorage.create(marksHostedRepo, marksAuthor, "test"); + var runner = new TestBotRunner(); + + var hgDir = tmp.path().resolve("hg"); + + var gitLocalDir = tmp.path().resolve("from.git"); + Files.createDirectories(gitLocalDir); + var gitLocalRepo = Repository.init(gitLocalDir, VCS.GIT); + populate(gitLocalRepo); + var gitHostedRepo = new TestHostedRepository(host, "from", gitLocalRepo); + + var bot = new CheckoutBot(gitHostedRepo.url(), gitLocalRepo.defaultBranch(), hgDir, storage, marksStorage); + runner.runPeriodicItems(bot); + + var hgRepo = Repository.get(hgDir).orElseThrow(); + assertEquals(3, hgRepo.commitMetadata().size()); + assertEquals(3, gitLocalRepo.commitMetadata().size()); + + var readme = gitLocalRepo.root().resolve("README"); + Files.write(readme, List.of("An updated line"), WRITE, APPEND); + gitLocalRepo.add(readme); + gitLocalRepo.commit("Updated Final README", "duke", "duke@openjdk.java.net"); + + runner.runPeriodicItems(bot); + assertEquals(4, hgRepo.commitMetadata().size()); + } + } +} diff --git a/bots/cli/build.gradle b/bots/cli/build.gradle index 95477f9b0..5ca4e9916 100644 --- a/bots/cli/build.gradle +++ b/bots/cli/build.gradle @@ -49,6 +49,7 @@ dependencies { implementation project(':bots:submit') implementation project(':bots:forward') implementation project(':bots:bridgekeeper') + implementation project(':bots:checkout') implementation project(':ci') implementation project(':vcs') implementation project(':jcheck') @@ -82,7 +83,8 @@ images { 'org.openjdk.skara.bots.tester', 'org.openjdk.skara.bots.topological', 'org.openjdk.skara.bots.forward', - 'org.openjdk.skara.bots.bridgekeeper'] + 'org.openjdk.skara.bots.bridgekeeper', + 'org.openjdk.skara.bots.checkout'] launchers = ['skara-bots': 'org.openjdk.skara.bots.cli/org.openjdk.skara.bots.cli.BotLauncher'] options = ["--module-path", "plugins"] bundles = ['zip', 'tar.gz'] diff --git a/settings.gradle b/settings.gradle index 083631561..3af4096b3 100644 --- a/settings.gradle +++ b/settings.gradle @@ -47,6 +47,7 @@ include 'issuetracker' include 'version' include 'bots:bridgekeeper' +include 'bots:checkout' include 'bots:cli' include 'bots:csr' include 'bots:forward'