bots: add checkout bot

bots/checkout/build.gradle

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+module {
+    name = 'org.openjdk.skara.bots.checkout'
+    test {
+        requires 'org.junit.jupiter.api'
+        requires 'org.openjdk.skara.test'
+        opens 'org.openjdk.skara.bots.checkout' to 'org.junit.platform.commons'
+    }
+}
+
+dependencies {
+    implementation project(':bot')
+    implementation project(':ci')
+    implementation project(':vcs')
+    implementation project(':host')
+    implementation project(':forge')
+    implementation project(':issuetracker')
+    implementation project(':census')
+    implementation project(':process')
+    implementation project(':json')
+    implementation project(':network')
+    implementation project(':storage')
+
+    testImplementation project(':test')
+}

bots/checkout/src/main/java/module-info.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+module org.openjdk.skara.bots.checkout {
+    requires org.openjdk.skara.vcs;
+    requires org.openjdk.skara.host;
+    requires org.openjdk.skara.network;
+    requires org.openjdk.skara.bot;
+    requires org.openjdk.skara.process;
+    requires org.openjdk.skara.storage;
+    requires java.logging;
+
+    provides org.openjdk.skara.bot.BotFactory with org.openjdk.skara.bots.checkout.CheckoutBotFactory;
+}

bots/checkout/src/main/java/org/openjdk/skara/bots/checkout/CheckoutBot.java

@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package org.openjdk.skara.bots.checkout;
+
+import org.openjdk.skara.bot.*;
+import org.openjdk.skara.vcs.*;
+import org.openjdk.skara.vcs.openjdk.convert.*;
+import org.openjdk.skara.storage.StorageBuilder;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.nio.file.*;
+import java.nio.charset.StandardCharsets;
+import java.net.URI;
+import java.net.URLEncoder;
+import java.util.logging.Logger;
+
+public class CheckoutBot implements Bot, WorkItem {
+    private static final Logger log = Logger.getLogger("org.openjdk.skara.bots");;
+    private final URI from;
+    private final Branch branch;
+    private final Path to;
+    private final Path storage;
+    private final StorageBuilder<Mark> marksStorage;
+
+    CheckoutBot(URI from, Branch branch, Path to, Path storage, StorageBuilder<Mark> marksStorage) {
+        this.from = from;
+        this.branch = branch;
+        this.to = to;
+        this.storage = storage;
+        this.marksStorage = marksStorage;
+    }
+
+    private static String urlEncode(Path p) {
+        return URLEncoder.encode(p.toString(), StandardCharsets.UTF_8);
+    }
+
+    private static String urlEncode(URI uri) {
+        return URLEncoder.encode(uri.toString(), StandardCharsets.UTF_8);
+    }
+
+    @Override
+    public boolean concurrentWith(WorkItem other) {
+        if (!(other instanceof CheckoutBot)) {
+            return true;
+        }
+        var o = (CheckoutBot) other;
+        return !(o.to.equals(to) || o.from.equals(from));
+    }
+
+    @Override
+    public String toString() {
+        return "CheckoutBot(" + from + ":" + branch.name() + ", " + to + ")";
+    }
+
+    @Override
+    public List<WorkItem> getPeriodicItems() {
+        return List.of(this);
+    }
+
+    @Override
+    public Collection<WorkItem> run(Path scratch) {
+        try {
+            var fromDir = storage.resolve(urlEncode(from));
+            Repository fromRepo = null;
+            if (!Files.exists(fromDir)) {
+                Files.createDirectories(fromDir);
+                log.info("Cloning Git repo " + from + " to " + fromDir);
+                fromRepo = Repository.clone(from, fromDir);
+            } else {
+                log.info("Getting existing Git repo repository from " + fromDir);
+                fromRepo = Repository.get(fromDir).orElseThrow(() ->
+                    new IllegalStateException("Git repository vanished from " + fromDir));
+            }
+            fromRepo.checkout(branch);
+            fromRepo.pull("origin", branch.name());
+
+            var repoName = Path.of(from.getPath()).getFileName().toString();
+            var marksDir = scratch.resolve("checkout").resolve("marks").resolve(repoName);
+            Files.createDirectories(marksDir);
+            var marks = marksStorage.materialize(marksDir);
+            var converter = new GitToHgConverter(branch);
+            try {
+                if (!Files.exists(to)) {
+                    log.info("Creating Hg repository at: " + to);
+                    Files.createDirectories(to);
+                    var toRepo = Repository.init(to, VCS.HG);
+                    converter.convert(fromRepo, toRepo);
+                } else {
+                    log.info("Found existing Hg repository at: " + to);
+                    var toRepo = Repository.get(to).orElseThrow(() ->
+                        new IllegalStateException("Repository vanished from " + to));
+                    var existing = new ArrayList<Mark>(marks.current());
+                    log.info("Found " + existing.size() + " existing marks");
+                    Collections.sort(existing);
+                    converter.convert(fromRepo, toRepo, existing);
+                }
+            } finally {
+                log.info("Storing " + converter.marks().size() + " marks");
+                marks.put(converter.marks());
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+        return List.of();
+    }
+}

bots/checkout/src/main/java/org/openjdk/skara/bots/checkout/CheckoutBotFactory.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.skara.bots.checkout;
+
+import org.openjdk.skara.bot.*;
+import org.openjdk.skara.vcs.*;
+import org.openjdk.skara.vcs.openjdk.convert.Mark;
+
+import java.util.*;
+import java.net.URI;
+import java.nio.file.Path;
+import java.util.logging.Logger;
+
+public class CheckoutBotFactory implements BotFactory {
+    private static final Logger log = Logger.getLogger("org.openjdk.skara.bots");
+
+    @Override
+    public String name() {
+        return "checkout";
+    }
+
+    @Override
+    public List<Bot> create(BotConfiguration configuration) {
+        var specific = configuration.specific();
+        var storage = configuration.storageFolder();
+
+        var marksRepo = configuration.repository(specific.get("marks").get("repo").asString());
+        var marksUser = Author.fromString(specific.get("marks").get("author").asString());
+
+        var bots = new ArrayList<Bot>();
+        for (var repo : specific.get("repositories").asArray()) {
+            var from = repo.get("from").asString();
+            var lastColon = from.lastIndexOf(":");
+            var fromURI = URI.create(from.substring(0, lastColon));
+            var fromBranch = new Branch(from.substring(lastColon + 1));
+            var to = Path.of(repo.get("to").asString());
+
+            var repoName = Path.of(fromURI.getPath()).getFileName().toString();
+            var markStorage = MarkStorage.create(marksRepo, marksUser, repoName);
+
+            bots.add(new CheckoutBot(fromURI, fromBranch, to, storage, markStorage));
+        }
+
+        return bots;
+    }
+}

bots/checkout/src/main/java/org/openjdk/skara/bots/checkout/MarkStorage.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package org.openjdk.skara.bots.checkout;
+
+import org.openjdk.skara.forge.HostedRepository;
+import org.openjdk.skara.vcs.Author;
+import org.openjdk.skara.vcs.Hash;
+import org.openjdk.skara.vcs.openjdk.convert.Mark;
+import org.openjdk.skara.storage.StorageBuilder;
+
+import java.net.URI;
+import java.util.*;
+import java.util.stream.Collectors;
+
+class MarkStorage {
+    private static Mark deserializeMark(String s) {
+        var parts = s.split(" ");
+        if (!(parts.length == 3 || parts.length == 4)) {
+            throw new IllegalArgumentException("Unexpected string:" + s);
+        }
+
+        var key = Integer.parseInt(parts[0]);
+        var hg = new Hash(parts[1]);
+        var git = new Hash(parts[2]);
+
+        return parts.length == 3 ? new Mark(key, hg, git) : new Mark(key, hg, git, new Hash(parts[3]));
+    }
+
+    private static String serialize(Collection<Mark> added, Set<Mark> existing) {
+        var marks = new ArrayList<Mark>();
+        var handled = new HashSet<Integer>();
+        for (var mark : added) {
+            marks.add(mark);
+            handled.add(mark.key());
+        }
+        for (var mark : existing) {
+            if (!handled.contains(mark.key())) {
+                marks.add(mark);
+            }
+        }
+        Collections.sort(marks);
+        var sb = new StringBuilder();
+        for (var mark : marks) {
+            sb.append(Integer.toString(mark.key()));
+            sb.append(" ");
+            sb.append(mark.hg().hex());
+            sb.append(" ");
+            sb.append(mark.git().hex());
+            if (mark.tag().isPresent()) {
+                sb.append(" ");
+                sb.append(mark.tag().get().hex());
+            }
+            sb.append("\n");
+        }
+        return sb.toString();
+    }
+
+    private static Set<Mark> deserialize(String current) {
+        var res = current.lines()
+                         .map(MarkStorage::deserializeMark)
+                         .collect(Collectors.toSet());
+        return res;
+    }
+
+    static StorageBuilder<Mark> create(HostedRepository repo, Author user, String name) {
+        return new StorageBuilder<Mark>(name + ".marks.txt")
+            .remoteRepository(repo, "master", user.name(), user.email(), "Updated marks for " + name)
+            .serializer(MarkStorage::serialize)
+            .deserializer(MarkStorage::deserialize);
+    }
+}