Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

57: The hgbridge should push marks to a configurable repository #44

Closed
wants to merge 1 commit into from
Closed

57: The hgbridge should push marks to a configurable repository #44

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion bots/hgbridge/src/main/java/org/openjdk/skara/bots/hgbridge/Exporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ private static void syncFolder(Path source, Path destination) throws IOException
}
}

static Optional<Repository> export(Converter converter, URI source, Path destination) throws IOException {
static Optional<Repository> export(Converter converter, URI source, Path destination, Path finalMarks) throws IOException {
final var successMarker = "success.txt";
final var lastKnownGood = destination.resolve("lkg");
final var current = destination.resolve("current");
Expand Down Expand Up @@ -199,6 +199,10 @@ static Optional<Repository> export(Converter converter, URI source, Path destina
Files.deleteIfExists(lastKnownGood.resolve(successMarker));
syncFolder(current, lastKnownGood);
lastKnownGood.resolve(successMarker).toFile().createNewFile();

// Update marks
var markSource = current.resolve("marks.txt");
Files.copy(markSource, finalMarks, StandardCopyOption.REPLACE_EXISTING);
}

return ret;
Expand Down
36 changes: 36 additions & 0 deletions bots/hgbridge/src/main/java/org/openjdk/skara/bots/hgbridge/ExporterConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ class ExporterConfig {
private URI source;
private HostedRepository configurationRepo;
private String configurationRef;
private HostedRepository marksRepo;
private String marksRef;
private String marksAuthorName;
private String marksAuthorEmail;
private List<String> replacementsFile;
private List<String> correctionsFile;
private List<String> lowercaseFile;
Expand Down Expand Up @@ -71,6 +75,38 @@ void configurationRef(String configurationRef) {
this.configurationRef = configurationRef;
}

void marksRepo(HostedRepository marksRepo) {
this.marksRepo = marksRepo;
}

HostedRepository marksRepo() {
return marksRepo;
}

void marksRef(String marksRef) {
this.marksRef = marksRef;
}

String marksRef() {
return marksRef;
}

void marksAuthorName(String marksAuthorName) {
this.marksAuthorName = marksAuthorName;
}

String marksAuthorName() {
return marksAuthorName;
}

void marksAuthorEmail(String marksAuthorEmail) {
this.marksAuthorEmail = marksAuthorEmail;
}

String marksAuthorEmail() {
return marksAuthorEmail;
}

void replacements(List<String> replacements) {
replacementsFile = replacements;
}
Expand Down
53 changes: 50 additions & 3 deletions bots/hgbridge/src/main/java/org/openjdk/skara/bots/hgbridge/JBridgeBot.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
package org.openjdk.skara.bots.hgbridge;

import org.openjdk.skara.bot.*;
import org.openjdk.skara.vcs.Repository;

import java.io.*;
import java.net.URLEncoder;
Expand Down Expand Up @@ -61,15 +62,61 @@ public boolean concurrentWith(WorkItem other) {
}
}

private void pushMarks(Path markSource, String destName, Path markScratchPath) throws IOException {
var marksRepo = Repository.materialize(markScratchPath, exporterConfig.marksRepo().getUrl(), exporterConfig.marksRef());

// We should never change existing marks
var markDest = markScratchPath.resolve(destName);
var updated = Files.readString(markSource);
if (Files.exists(markDest)) {
var existing = Files.readString(markDest);

if (!updated.startsWith(existing)) {
throw new RuntimeException("Update containing conflicting marks!");
}
if (existing.equals(updated)) {
// Nothing new to push
return;
}
} else {
if (!Files.exists(markDest.getParent())) {
Files.createDirectories(markDest.getParent());
}
}

Files.writeString(markDest, updated, StandardCharsets.UTF_8);
marksRepo.add(markDest);
var hash = marksRepo.commit("Updated marks", exporterConfig.marksAuthorName(), exporterConfig.marksAuthorEmail());
marksRepo.push(hash, exporterConfig.marksRepo().getUrl(), exporterConfig.marksRef());
}

@Override
public void run(Path scratchPath) {
log.fine("Running export for " + exporterConfig.source().toString());

try {
var converter = exporterConfig.resolve(scratchPath);
var exported = Exporter.export(converter, exporterConfig.source(), storage);
IOException lastException = null;
var converter = exporterConfig.resolve(scratchPath.resolve("converter"));
var marksFile = scratchPath.resolve("marks.txt");
var exported = Exporter.export(converter, exporterConfig.source(), storage, marksFile);

// Push updated marks - other marks files may be updated concurrently, so try a few times
var retryCount = 0;
while (exported.isPresent()) {
try {
pushMarks(marksFile,
exporterConfig.source().getHost() + "/" + exporterConfig.source().getPath() + "/marks.txt",
scratchPath.resolve("markspush"));
break;
} catch (IOException e) {
retryCount++;
if (retryCount > 10) {
log.warning("Retry count exceeded for pushing marks");
throw new UncheckedIOException(e);
}
}
}

IOException lastException = null;
for (var destination : exporterConfig.destinations()) {
var markerBase = destination.getUrl().getHost() + "/" + destination.getName();
var successfulPushMarker = storage.resolve(URLEncoder.encode(markerBase, StandardCharsets.UTF_8) + ".success.txt");
Expand Down
12 changes: 12 additions & 0 deletions bots/hgbridge/src/main/java/org/openjdk/skara/bots/hgbridge/JBridgeBotFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@ public List<Bot> create(BotConfiguration configuration) {
var specific = configuration.specific();
var storage = configuration.storageFolder();

var marks = specific.get("marks").asObject();
var marksRepo = configuration.repository(marks.get("repository").asString());
var marksRef = marks.get("ref").asString();
var marksName = marks.get("name").asString();
var marksEmail = marks.get("email").asString();

var converters = specific.get("converters").stream()
.map(JSONValue::asObject)
.flatMap(base -> base.get("repositories").stream()
Expand All @@ -62,6 +68,12 @@ public List<Bot> create(BotConfiguration configuration) {
converterConfig.configurationRepo(configuration.repository(base.get("repository").asString()));
converterConfig.configurationRef(base.get("ref").asString());

// Mark storage configuration
converterConfig.marksRepo(marksRepo);
converterConfig.marksRef(marksRef);
converterConfig.marksAuthorName(marksName);
converterConfig.marksAuthorEmail(marksEmail);

// Repository specific overrides
converterConfig.replacements(getSpecific("replacements", base, repo));
converterConfig.corrections(getSpecific("corrections", base, repo));
Expand Down
88 changes: 72 additions & 16 deletions bots/hgbridge/src/test/java/org/openjdk/skara/bots/hgbridge/BridgeBotTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
*/
package org.openjdk.skara.bots.hgbridge;

import org.openjdk.skara.host.HostedRepository;
import org.openjdk.skara.host.*;
import org.openjdk.skara.host.network.URIBuilder;
import org.openjdk.skara.process.Process;
import org.openjdk.skara.test.*;
Expand Down Expand Up @@ -63,9 +63,26 @@ private List<String> runHgCommand(Repository repository, String... params) throw
static class TestExporterConfig extends ExporterConfig {
private boolean badAuthors = false;

TestExporterConfig(URI source, HostedRepository destination) {
TestExporterConfig(URI source, HostedRepository destination, Path marksRepoPath) throws IOException {
this.source(source);
this.destinations(List.of(destination));

var host = TestHost.createNew(List.of(new HostUserDetails(0, "duke", "J. Duke")));
var marksLocalRepo = Repository.init(marksRepoPath.resolve("marks.git"), VCS.GIT);

var initialFile = marksLocalRepo.root().resolve("init.txt");
if (!Files.exists(initialFile)) {
Files.writeString(initialFile, "Hello", StandardCharsets.UTF_8);
marksLocalRepo.add(initialFile);
var hash = marksLocalRepo.commit("First", "duke", "duke@duke.duke");
marksLocalRepo.checkout(hash, true); // Have to move away from the master branch to allow pushes
}

var marksHostedRepo = new TestHostedRepository(host, "test", marksLocalRepo);
this.marksRepo(marksHostedRepo);
this.marksRef("master");
this.marksAuthorName("J. Duke");
this.marksAuthorEmail("j@duke.duke");
}

void setBadAuthors() {
Expand Down Expand Up @@ -139,12 +156,13 @@ void bridgeTest(TestInfo testInfo) throws IOException {
var hgFolder = new TemporaryDirectory();
var gitFolder = new TemporaryDirectory();
var storageFolder = new TemporaryDirectory();
var storageFolder2 = new TemporaryDirectory()) {
var storageFolder2 = new TemporaryDirectory();
var marksFolder = new TemporaryDirectory()) {
// Export a partial version of a hg repository
var localHgRepo = Repository.materialize(hgFolder.path(), source, "default");
localHgRepo.fetch(source, "testlock");
var destinationRepo = credentials.getHostedRepository();
var config = new TestExporterConfig(localHgRepo.root().toUri(), destinationRepo);
var config = new TestExporterConfig(localHgRepo.root().toUri(), destinationRepo, marksFolder.path());
var bridge = new JBridgeBot(config, storageFolder.path());

runHgCommand(localHgRepo, "strip", "-r", "bd7a3ed1210f");
Expand Down Expand Up @@ -188,11 +206,12 @@ void bridgeTest(TestInfo testInfo) throws IOException {
void bridgeCorruptedStorageHg(TestInfo testInfo) throws IOException {
try (var credentials = new HostCredentials(testInfo);
var storageFolder = new TemporaryDirectory();
var gitFolder = new TemporaryDirectory()) {
var gitFolder = new TemporaryDirectory();
var marksFolder = new TemporaryDirectory()) {
var destinationRepo = credentials.getHostedRepository();

// Export an hg repository as is
var config = new TestExporterConfig(source, destinationRepo);
var config = new TestExporterConfig(source, destinationRepo, marksFolder.path());
var bridge = new JBridgeBot(config, storageFolder.path());
TestBotRunner.runPeriodicItems(bridge);

Expand Down Expand Up @@ -226,17 +245,18 @@ void bridgeExportScriptFailure(TestInfo testInfo) throws IOException {
try (var credentials = new HostCredentials(testInfo);
var storageFolder = new TemporaryDirectory();
var storageFolder2 = new TemporaryDirectory();
var gitFolder = new TemporaryDirectory()) {
var gitFolder = new TemporaryDirectory();
var marksFolder = new TemporaryDirectory()) {
var destinationRepo = credentials.getHostedRepository();

// Export an hg repository but with an empty authors list
var config = new TestExporterConfig(source, destinationRepo);
var config = new TestExporterConfig(source, destinationRepo, marksFolder.path());
config.setBadAuthors();
var badBridge = new JBridgeBot(config, storageFolder.path());
assertThrows(RuntimeException.class, () -> TestBotRunner.runPeriodicItems(badBridge));

// Now once again with a correct configuration
config = new TestExporterConfig(source, destinationRepo);
config = new TestExporterConfig(source, destinationRepo, marksFolder.path());
var goodBridge = new JBridgeBot(config, storageFolder2.path());
TestBotRunner.runPeriodicItems(goodBridge);

Expand All @@ -254,9 +274,10 @@ void bridgeReuseMarks(TestInfo testInfo) throws IOException {
var gitFolder = new TemporaryDirectory();
var gitFolder2 = new TemporaryDirectory();
var gitFolder3 = new TemporaryDirectory();
var gitFolder4 = new TemporaryDirectory()) {
var gitFolder4 = new TemporaryDirectory();
var marksFolder = new TemporaryDirectory()) {
var destinationRepo = credentials.getHostedRepository();
var config = new TestExporterConfig(source, destinationRepo);
var config = new TestExporterConfig(source, destinationRepo, marksFolder.path());

// Export an hg repository as is
var bridge = new JBridgeBot(config, storageFolder.path());
Expand Down Expand Up @@ -295,9 +316,10 @@ void retryFailedPush(TestInfo testInfo) throws IOException {
var gitFolder2 = new TemporaryDirectory();
var gitFolder3 = new TemporaryDirectory();
var gitFolder4 = new TemporaryDirectory();
var gitFolder5 = new TemporaryDirectory()) {
var gitFolder5 = new TemporaryDirectory();
var marksFolder = new TemporaryDirectory()) {
var destinationRepo = credentials.getHostedRepository();
var config = new TestExporterConfig(source, destinationRepo);
var config = new TestExporterConfig(source, destinationRepo, marksFolder.path());

// Export an hg repository as is
var bridge = new JBridgeBot(config, storageFolder.path());
Expand Down Expand Up @@ -351,14 +373,13 @@ void retryFailedPush(TestInfo testInfo) throws IOException {
void filterUnreachable(TestInfo testInfo) throws IOException {
try (var credentials = new HostCredentials(testInfo);
var hgFolder = new TemporaryDirectory();
var gitFolder = new TemporaryDirectory();
var storageFolder = new TemporaryDirectory();
var storageFolder2 = new TemporaryDirectory()) {
var marksFolder = new TemporaryDirectory()) {
// Export a hg repository with unreachable commits
var localHgRepo = Repository.materialize(hgFolder.path(), source, "default");
localHgRepo.fetch(source, "testlock");
var destinationRepo = credentials.getHostedRepository();
var config = new TestExporterConfig(localHgRepo.root().toUri(), destinationRepo);
var config = new TestExporterConfig(localHgRepo.root().toUri(), destinationRepo, marksFolder.path());
var bridge = new JBridgeBot(config, storageFolder.path());

runHgCommand(localHgRepo, "update", "-r", "5");
Expand All @@ -375,4 +396,39 @@ void filterUnreachable(TestInfo testInfo) throws IOException {
TestBotRunner.runPeriodicItems(bridge);
}
}

@Test
void changedMarks(TestInfo testInfo) throws IOException {
try (var credentials = new HostCredentials(testInfo);
var hgFolder = new TemporaryDirectory();
var storageFolder = new TemporaryDirectory();
var storageFolder2 = new TemporaryDirectory();
var marksFolder = new TemporaryDirectory()) {
// Export a hg repository
var localHgRepo = Repository.materialize(hgFolder.path(), source, "default");
localHgRepo.fetch(source, "testlock");
var destinationRepo = credentials.getHostedRepository();
var config = new TestExporterConfig(localHgRepo.root().toUri(), destinationRepo, marksFolder.path());
var bridge = new JBridgeBot(config, storageFolder.path());

runHgCommand(localHgRepo, "update", "-r", "5");
var other = localHgRepo.root().resolve("other.txt");
Files.writeString(other, "Hello");
localHgRepo.add(other);
localHgRepo.commit("First", "duke", "");

// Do an initial conversion
TestBotRunner.runPeriodicItems(bridge);

// Now roll back and commit something else
runHgCommand(localHgRepo, "update", "-r", "5");
Files.writeString(other, "There");
localHgRepo.add(other);
localHgRepo.commit("Second", "duke", "");

// The second conversion (with fresh storage) should detect that marks have changed
var newBridge = new JBridgeBot(config, storageFolder2.path());
assertThrows(RuntimeException.class, () -> TestBotRunner.runPeriodicItems(newBridge));
}
}
}
2 changes: 1 addition & 1 deletion vcs/src/main/java/org/openjdk/skara/vcs/openjdk/convert/HgToGitConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -686,7 +686,7 @@ public List<Mark> pull(Repository hgRepo, URI source, Repository gitRepo, List<M
for (var mark : marks) {
hgHashesToMarks.put(mark.hg(), mark.key());
marksToHgHashes.put(mark.key(), mark.hg());
currentMark = mark.key() > currentMark ? mark.key() : currentMark;
currentMark = Math.max(mark.key(), currentMark);
}
var gitMarks = writeMarks(marks);
convert(hg, git, hgRepo, gitMarks);
Expand Down