Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow a HostedRepositoryPool to refresh the seed from upstream #874

Closed
wants to merge 3 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -26,14 +26,12 @@
import org.openjdk.skara.forge.*;
import org.openjdk.skara.host.HostUser;
import org.openjdk.skara.jcheck.JCheckConfiguration;
import org.openjdk.skara.vcs.Repository;

import java.io.*;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.*;
import java.util.Optional;
import java.util.stream.Collectors;

class CensusInstance {
private final Census census;
@@ -48,14 +46,6 @@ private CensusInstance(Census census, JCheckConfiguration configuration, Project
this.namespace = namespace;
}

private static Repository initialize(HostedRepository repo, String ref, Path folder) {
try {
return Repository.materialize(folder, repo.url(), "+" + ref + ":pr_census_" + repo.name());
} catch (IOException e) {
throw new RuntimeException("Failed to retrieve census to " + folder, e);
}
}

private static Project project(JCheckConfiguration configuration, Census census) {
var project = census.project(configuration.general().project());

@@ -76,31 +66,31 @@ private static Namespace namespace(Census census, String hostNamespace) {
return namespace;
}

private static JCheckConfiguration configuration(HostedRepository remoteRepo, String name, String ref) {
var confFile = remoteRepo.fileContents(name, ref);
return JCheckConfiguration.parse(confFile.lines().collect(Collectors.toList()));
private static JCheckConfiguration configuration(HostedRepositoryPool hostedRepositoryPool, HostedRepository remoteRepo, String name, String ref, Path folder) throws IOException {
var repoName = remoteRepo.url().getHost() + "/" + remoteRepo.name();
var repoFolder = folder.resolve(URLEncoder.encode(repoName, StandardCharsets.UTF_8));
hostedRepositoryPool.checkoutAllowStale(remoteRepo, ref, repoFolder);

var confFile = Files.readAllLines(repoFolder.resolve(name));
return JCheckConfiguration.parse(confFile);
}

static CensusInstance create(HostedRepository censusRepo, String censusRef, Path folder, PullRequest pr,
static CensusInstance create(HostedRepositoryPool hostedRepositoryPool, HostedRepository censusRepo, String censusRef, Path folder, PullRequest pr,
HostedRepository confOverrideRepo, String confOverrideName, String confOverrideRef) {
var repoName = censusRepo.url().getHost() + "/" + censusRepo.name();
var repoFolder = folder.resolve(URLEncoder.encode(repoName, StandardCharsets.UTF_8));
try {
var localRepo = Repository.get(repoFolder)
.or(() -> Optional.of(initialize(censusRepo, censusRef, repoFolder)))
.orElseThrow();
var hash = localRepo.fetch(censusRepo.url(), censusRef, false);
localRepo.checkout(hash, true);
hostedRepositoryPool.checkoutAllowStale(censusRepo, censusRef, repoFolder);
} catch (IOException e) {
initialize(censusRepo, censusRef, repoFolder);
throw new UncheckedIOException("Cannot materialize census to " + repoFolder, e);
}

try {
JCheckConfiguration configuration;
if (confOverrideRepo == null) {
configuration = configuration(pr.repository(), ".jcheck/conf", pr.targetRef());
configuration = configuration(hostedRepositoryPool, pr.repository(), ".jcheck/conf", pr.targetRef(), folder);
} else {
configuration = configuration(confOverrideRepo, confOverrideName, confOverrideRef);
configuration = configuration(hostedRepositoryPool, confOverrideRepo, confOverrideName, confOverrideRef, folder);
}
var census = Census.parse(repoFolder);
var project = project(configuration, census);
@@ -175,7 +175,10 @@ public String toString() {
@Override
public Collection<WorkItem> run(Path scratchPath) {
// First determine if the current state of the PR has already been checked
var census = CensusInstance.create(bot.censusRepo(), bot.censusRef(), scratchPath.resolve("census"), pr,
var seedPath = bot.seedStorage().orElse(scratchPath.resolve("seeds"));
var hostedRepositoryPool = new HostedRepositoryPool(seedPath);

var census = CensusInstance.create(hostedRepositoryPool, bot.censusRepo(), bot.censusRef(), scratchPath.resolve("census"), pr,
bot.confOverrideRepository().orElse(null), bot.confOverrideName(), bot.confOverrideRef());
var comments = pr.comments();
var allReviews = pr.reviews();
@@ -244,8 +247,6 @@ public Collection<WorkItem> run(Path scratchPath) {
}

try {
var seedPath = bot.seedStorage().orElse(scratchPath.resolve("seeds"));
var hostedRepositoryPool = new HostedRepositoryPool(seedPath);
var localRepoPath = scratchPath.resolve("pr").resolve("check").resolve(pr.repository().name());
var localRepo = PullRequestUtils.materialize(hostedRepositoryPool, pr, localRepoPath);

@@ -23,8 +23,7 @@
package org.openjdk.skara.bots.pr;

import org.openjdk.skara.bot.WorkItem;
import org.openjdk.skara.forge.PullRequest;
import org.openjdk.skara.forge.PullRequestBody;
import org.openjdk.skara.forge.*;
import org.openjdk.skara.host.HostUser;
import org.openjdk.skara.issuetracker.Comment;

@@ -222,7 +221,10 @@ public Collection<WorkItem> run(Path scratchPath) {
return List.of(new LabelerWorkItem(bot, updatedPR, errorHandler));
}

var census = CensusInstance.create(bot.censusRepo(), bot.censusRef(), scratchPath.resolve("census"), pr,
var seedPath = bot.seedStorage().orElse(scratchPath.resolve("seeds"));
var hostedRepositoryPool = new HostedRepositoryPool(seedPath);

var census = CensusInstance.create(hostedRepositoryPool, bot.censusRepo(), bot.censusRef(), scratchPath.resolve("census"), pr,
bot.confOverrideRepository().orElse(null), bot.confOverrideName(), bot.confOverrideRef());
var command = nextCommand.get();
log.info("Processing command: " + command.id() + " - " + command.name());
@@ -50,10 +50,12 @@ void simpleCommit(TestInfo testInfo) throws IOException {
var censusBuilder = credentials.getCensusBuilder()
.addAuthor(author.forge().currentUser().id())
.addReviewer(reviewer.forge().currentUser().id());
var seedFolder = tempFolder.path().resolve("seed");
var checkBot = PullRequestBot.newBuilder()
.repo(author)
.censusRepo(censusBuilder.build())
.censusLink("https://census.com/{{contributor}}-profile")
.seedStorage(seedFolder)
.build();

// Populate the projects repository
@@ -26,6 +26,7 @@

import java.io.*;
import java.nio.file.*;
import java.time.*;
import java.util.*;
import java.util.logging.Logger;

@@ -57,23 +58,46 @@ private void clearDirectory(Path directory) {
}
}

private void initializeSeed() throws IOException {
private void refreshSeed(boolean allowStale) throws IOException {
if (!Files.exists(seed)) {
Files.createDirectories(seed.getParent());
var tmpSeedFolder = seed.resolveSibling(seed.getFileName().toString() + "-" + UUID.randomUUID());
Repository.clone(hostedRepository.url(), tmpSeedFolder, true);
try {
Files.move(tmpSeedFolder, seed);
log.info("Seeded repository " + hostedRepository.name() + " into " + seed);
return;
} catch (IOException e) {
log.info("Failed to populate seed folder " + seed + " - perhaps due to a benign race. Ignoring..");
clearDirectory(tmpSeedFolder);
}
}

var seedRepo = Repository.get(seed).orElseThrow(() -> new IOException("Existing seed is corrupt?"));
if (allowStale) {
try {
var lastFetch = Files.getLastModifiedTime(seed.resolve("FETCH_HEAD"));
if (lastFetch.toInstant().isAfter(Instant.now().minus(Duration.ofMinutes(1)))) {
log.info("Seed should be up to date, skipping fetch");
return;
}
} catch (IOException ignored) {
}
log.info("Seed is potentially stale, time to fetch the latest upstream changes");
} else {
log.info("Fetching latest upstream changes into the seed");
}
try {
seedRepo.fetchAll();
} catch (IOException e) {
if (!allowStale) {
throw e;
}
}
}

private Repository cloneSeeded(Path path) throws IOException {
initializeSeed();
private Repository cloneSeeded(Path path, boolean allowStale) throws IOException {
refreshSeed(allowStale);
log.info("Using seed folder " + seed + " when cloning into " + path);
return Repository.clone(hostedRepository.url(), path, false, seed);
}
@@ -95,23 +119,25 @@ private void removeOldClone(Path path, String reason) {
}
}

private Repository materializeClone(Path path) throws IOException {
private Repository materializeClone(Path path, boolean allowStale) throws IOException {
var localRepo = Repository.get(path);
if (localRepo.isEmpty()) {
removeOldClone(path, "norepo");
return cloneSeeded(path);
return cloneSeeded(path, allowStale);
} else {
var localRepoInstance = localRepo.get();
if (!localRepoInstance.isHealthy()) {
removeOldClone(path, "unhealthy");
return cloneSeeded(path);
return cloneSeeded(path, allowStale);
} else {
try {
refreshSeed(allowStale);
localRepoInstance.clean();
localRepoInstance.fetchAll();
return localRepoInstance;
} catch (IOException e) {
removeOldClone(path, "uncleanable");
return cloneSeeded(path);
return cloneSeeded(path, allowStale);
}
}
}
@@ -120,47 +146,30 @@ private Repository materializeClone(Path path) throws IOException {

public Repository materialize(HostedRepository hostedRepository, Path path) throws IOException {
var hostedRepositoryInstance = new HostedRepositoryInstance(hostedRepository);
return hostedRepositoryInstance.materializeClone(path);
return hostedRepositoryInstance.materializeClone(path, false);
}

private static class NewClone {
private final Repository repository;
private final Hash fetchHead;

NewClone(Repository repository, Hash fetchHead) {
this.repository = repository;
this.fetchHead = fetchHead;
}

Repository repository() {
return repository;
}

Hash fetchHead() {
return fetchHead;
}
}

private NewClone fetchRef(HostedRepository hostedRepository, Repository repository, String ref) throws IOException {
var fetchHead = repository.fetch(hostedRepository.url(), "+" + ref + ":hostedrepositorypool");
return new NewClone(repository, fetchHead);
}

public Repository checkout(HostedRepository hostedRepository, String ref, Path path) throws IOException {
private Repository checkout(HostedRepository hostedRepository, String ref, Path path, boolean allowStale) throws IOException {
var hostedRepositoryInstance = new HostedRepositoryInstance(hostedRepository);
var clone = fetchRef(hostedRepository, hostedRepositoryInstance.materializeClone(path), ref);
var localRepo = clone.repository();
var localClone = hostedRepositoryInstance.materializeClone(path, allowStale);
try {
localRepo.checkout(clone.fetchHead(), true);
localClone.checkout(new Branch(ref), true);
} catch (IOException e) {
var preserveUnchecked = hostedRepositoryInstance.seed.resolveSibling(
hostedRepositoryInstance.seed.getFileName().toString() + "-unchecked-" + UUID.randomUUID());
log.severe("Uncheckoutable local repository detected - preserved in: " + preserveUnchecked);
Files.move(localRepo.root(), preserveUnchecked);
clone = fetchRef(hostedRepository, hostedRepositoryInstance.cloneSeeded(path), ref);
localRepo = clone.repository();
localRepo.checkout(clone.fetchHead(), true);
Files.move(localClone.root(), preserveUnchecked);
localClone = hostedRepositoryInstance.materializeClone(path, allowStale);
localClone.checkout(new Branch(ref), true);
}
return localRepo;
return localClone;
}

public Repository checkout(HostedRepository hostedRepository, String ref, Path path) throws IOException {
return checkout(hostedRepository, ref, path, false);
}

public Repository checkoutAllowStale(HostedRepository hostedRepository, String ref, Path path) throws IOException {
return checkout(hostedRepository, ref, path, true);
}
}