Skip to content
Permalink
Browse files
301: Keep track of branch progress per notifier
Reviewed-by: ehelin
  • Loading branch information
rwestberg committed Mar 13, 2020
1 parent d0eb797 commit 286ccf84f0440c9808a57ed80f7d1f3593c126e5
Show file tree
Hide file tree
Showing 9 changed files with 269 additions and 82 deletions.
@@ -356,6 +356,11 @@ public boolean isIdempotent() {
return true;
}

@Override
public String name() {
return "issue";
}

@Override
public void handleNewIssue(PullRequest pr, org.openjdk.skara.vcs.openjdk.Issue issue) {
var realIssue = issueProject.issue(issue.id());
@@ -111,4 +111,9 @@ public void handleNewBranch(HostedRepository repository, Repository localReposit
public boolean isIdempotent() {
return false;
}

@Override
public String name() {
return "json";
}
}
@@ -384,4 +384,9 @@ public void handleNewBranch(HostedRepository repository, Repository localReposit
public boolean isIdempotent() {
return false;
}

@Override
public String name() {
return "ml";
}
}
@@ -34,4 +34,5 @@ public interface RepositoryUpdateConsumer {
void handleTagCommit(HostedRepository repository, Repository localRepository, Commit commit, Tag tag, Tag.Annotated annotation);
void handleNewBranch(HostedRepository repository, Repository localRepository, List<Commit> commits, Branch parent, Branch branch);
boolean isIdempotent();
String name();
}
@@ -54,7 +54,7 @@ public class RepositoryWorkItem implements WorkItem {
this.updaters = updaters;
}

private void handleNewRef(Repository localRepo, Reference ref, Collection<Reference> allRefs, boolean runOnlyIdempotent) {
private void handleNewRef(Repository localRepo, Reference ref, Collection<Reference> allRefs, RepositoryUpdateConsumer updater) {
// Figure out the best parent ref
var candidates = new HashSet<>(allRefs);
candidates.remove(ref);
@@ -77,55 +77,68 @@ private void handleNewRef(Repository localRepo, Reference ref, Collection<Refere
throw new RuntimeException("Excessive amount of unique commits on new branch " + ref.name() +
" detected (" + bestParent.getValue().size() + ") - skipping notifications");
}
List<Commit> bestParentCommits;
try {
var bestParentCommits = localRepo.commits(bestParent.getKey().hash().hex() + ".." + ref.hash(), true);
for (var updater : updaters) {
if (updater.isIdempotent() != runOnlyIdempotent) {
continue;
}
var branch = new Branch(ref.name());
var parent = new Branch(bestParent.getKey().name());
updater.handleNewBranch(repository, localRepo, bestParentCommits.asList(), parent, branch);
}
bestParentCommits = localRepo.commits(bestParent.getKey().hash().hex() + ".." + ref.hash(), true).asList();
} catch (IOException e) {
e.printStackTrace();
throw new UncheckedIOException(e);
}
var branch = new Branch(ref.name());
var parent = new Branch(bestParent.getKey().name());
updater.handleNewBranch(repository, localRepo, bestParentCommits, parent, branch);
}

private void handleUpdatedRef(Repository localRepo, Reference ref, List<Commit> commits, boolean runOnlyIdempotent) {
for (var updater : updaters) {
if (updater.isIdempotent() != runOnlyIdempotent) {
continue;
}
var branch = new Branch(ref.name());
updater.handleCommits(repository, localRepo, commits, branch);
}
private void handleUpdatedRef(Repository localRepo, Reference ref, List<Commit> commits, RepositoryUpdateConsumer updater) {
var branch = new Branch(ref.name());
updater.handleCommits(repository, localRepo, commits, branch);
}

private void handleRef(Repository localRepo, UpdateHistory history, Reference ref, Collection<Reference> allRefs) throws IOException {
private List<RuntimeException> handleRef(Repository localRepo, UpdateHistory history, Reference ref, Collection<Reference> allRefs) throws IOException {
var errors = new ArrayList<RuntimeException>();
var branch = new Branch(ref.name());
var lastHash = history.branchHash(branch);
if (lastHash.isEmpty()) {
log.warning("No previous history found for branch '" + branch + "' - resetting mark");
handleNewRef(localRepo, ref, allRefs, true);
history.setBranchHash(branch, ref.hash());
handleNewRef(localRepo, ref, allRefs, false);
} else {
var commitMetadata = localRepo.commitMetadata(lastHash.get() + ".." + ref.hash());
if (commitMetadata.size() == 0) {
return;
}
if (commitMetadata.size() > 1000) {
history.setBranchHash(branch, ref.hash());
throw new RuntimeException("Excessive amount of new commits on branch " + branch.name() +
" detected (" + commitMetadata.size() + ") - skipping notifications");
}
for (var updater : updaters) {
var lastHash = history.branchHash(branch, updater.name());
if (lastHash.isEmpty()) {
log.warning("No previous history found for branch '" + branch + "' and updater '" + updater.name() + " - resetting mark");
if (!updater.isIdempotent()) {
history.setBranchHash(branch, updater.name(), ref.hash());
}
try {
handleNewRef(localRepo, ref, allRefs, updater);
if (updater.isIdempotent()) {
history.setBranchHash(branch, updater.name(), ref.hash());
}
} catch (RuntimeException e) {
errors.add(e);
}
} else {
var commitMetadata = localRepo.commitMetadata(lastHash.get() + ".." + ref.hash());
if (commitMetadata.size() == 0) {
continue;
}
if (commitMetadata.size() > 1000) {
history.setBranchHash(branch, updater.name(), ref.hash());
errors.add(new RuntimeException("Excessive amount of new commits on branch " + branch.name() +
" detected (" + commitMetadata.size() + ") for updater '" +
updater.name() + "' - skipping notifications"));
continue;
}

var commits = localRepo.commits(lastHash.get() + ".." + ref.hash(), true).asList();
handleUpdatedRef(localRepo, ref, commits, true);
history.setBranchHash(branch, ref.hash());
handleUpdatedRef(localRepo, ref, commits, false);
var commits = localRepo.commits(lastHash.get() + ".." + ref.hash(), true).asList();
if (!updater.isIdempotent()) {
history.setBranchHash(branch, updater.name(), ref.hash());
}
try {
handleUpdatedRef(localRepo, ref, commits, updater);
if (updater.isIdempotent()) {
history.setBranchHash(branch, updater.name(), ref.hash());
}
} catch (RuntimeException e) {
errors.add(e);
}
}
}
return errors;
}

private Optional<OpenJDKTag> existingPrevious(OpenJDKTag tag, Set<OpenJDKTag> allJdkTags) {
@@ -276,17 +289,23 @@ public void run(Path scratchPath) {
var history = UpdateHistory.create(tagStorageBuilder, historyPath.resolve("tags"), branchStorageBuilder, historyPath.resolve("branches"));
handleTags(localRepo, history);

boolean hasBranchHistory = knownRefs.stream()
.map(ref -> history.branchHash(new Branch(ref.name())))
.anyMatch(Optional::isPresent);
boolean hasBranchHistory = !history.isEmpty();
var errors = new ArrayList<RuntimeException>();
for (var ref : knownRefs) {
if (!hasBranchHistory) {
log.warning("No previous history found for any branch - resetting mark for '" + ref.name() + "'");
history.setBranchHash(new Branch(ref.name()), ref.hash());
log.warning("No previous history found for any branch - resetting mark for '" + ref.name());
for (var updater : updaters) {
log.info("Resetting mark for branch '" + ref.name() + "' for updater '" + updater.name() + "'");
history.setBranchHash(new Branch(ref.name()), updater.name(), ref.hash());
}
} else {
handleRef(localRepo, history, ref, knownRefs);
errors.addAll(handleRef(localRepo, history, ref, knownRefs));
}
}
if (!errors.isEmpty()) {
errors.forEach(error -> log.throwing("RepositoryWorkItem", "run", error));
throw new RuntimeException("Errors detected during ref updating");
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
@@ -28,24 +28,25 @@

class ResolvedBranch {
private final Branch branch;
private final String updater;
private final Hash hash;

ResolvedBranch(Branch branch, Hash hash) {
ResolvedBranch(Branch branch, String updater, Hash hash) {
this.branch = branch;
this.updater = updater;
this.hash = hash;
}

public Branch branch() {
return branch;
}

public Hash hash() {
return hash;
public String updater() {
return updater;
}

@Override
public String toString() {
return branch.name() + ":" + hash().hex();
public Hash hash() {
return hash;
}

@Override
@@ -57,12 +58,11 @@ public boolean equals(Object o) {
return false;
}
ResolvedBranch that = (ResolvedBranch) o;
return Objects.equals(branch, that.branch) &&
Objects.equals(hash, that.hash);
return branch.equals(that.branch) && updater.equals(that.updater) && hash.equals(that.hash);
}

@Override
public int hashCode() {
return Objects.hash(branch, hash);
return Objects.hash(branch, updater, hash);
}
}
@@ -27,30 +27,44 @@

import java.nio.file.Path;
import java.util.*;
import java.util.function.Function;
import java.util.stream.*;

class UpdateHistory {

private final Storage<Tag> tagStorage;
private final Storage<ResolvedBranch> branchStorage;

private Map<Branch, Hash> branches;
private Map<String, Hash> branchHashes;
private Set<Tag> tags;

private List<ResolvedBranch> parseSerializedEntry(String entry) {
var parts = entry.split(" ");
if (parts.length == 2) {
// Transform legacy entry
var issueEntry = new ResolvedBranch(new Branch(parts[0]), "issue", new Hash(parts[1]));
var mlEntry = new ResolvedBranch(new Branch(parts[0]), "ml", new Hash(parts[1]));
return List.of(issueEntry, mlEntry);
}
return List.of(new ResolvedBranch(new Branch(parts[0]), parts[1], new Hash(parts[2])));
}

private Set<ResolvedBranch> loadBranches(String current) {
return current.lines()
.map(line -> line.split(" "))
.map(entry -> new ResolvedBranch(new Branch(entry[0]), new Hash(entry[1])))
.flatMap(line -> parseSerializedEntry(line).stream())
.collect(Collectors.toSet());
}

private String serializeEntry(ResolvedBranch entry) {
return entry.branch().toString() + " " + entry.updater() + " " + entry.hash().toString();
}

private String serializeBranches(Collection<ResolvedBranch> added, Set<ResolvedBranch> existing) {
var updatedBranches = existing.stream()
.collect(Collectors.toMap(ResolvedBranch::branch,
ResolvedBranch::hash));
added.forEach(a -> updatedBranches.put(a.branch(), a.hash()));
return updatedBranches.entrySet().stream()
.map(entry -> entry.getKey().toString() + " " + entry.getValue().toString())
.collect(Collectors.toMap(entry -> entry.branch().toString() + ":" + entry.updater(),
Function.identity()));
added.forEach(a -> updatedBranches.put(a.branch().toString() + ":" + a.updater(), a));
return updatedBranches.values().stream()
.map(this::serializeEntry)
.sorted()
.collect(Collectors.joining("\n"));
}
@@ -73,9 +87,9 @@ private Set<Tag> currentTags() {
return tagStorage.current();
}

private Map<Branch, Hash> currentBranchHashes() {
private Map<String, Hash> currentBranchHashes() {
return branchStorage.current().stream()
.collect(Collectors.toMap(ResolvedBranch::branch, ResolvedBranch::hash));
.collect(Collectors.toMap(rb -> rb.branch().toString() + " " + rb.updater(), ResolvedBranch::hash));
}

private UpdateHistory(StorageBuilder<Tag> tagStorageBuilder, Path tagLocation, StorageBuilder<ResolvedBranch> branchStorageBuilder, Path branchLocation) {
@@ -90,7 +104,7 @@ private UpdateHistory(StorageBuilder<Tag> tagStorageBuilder, Path tagLocation, S
.materialize(branchLocation);

tags = currentTags();
branches = currentBranchHashes();
branchHashes = currentBranchHashes();
}

static UpdateHistory create(StorageBuilder<Tag> tagStorageBuilder, Path tagLocation, StorageBuilder<ResolvedBranch> branchStorageBuilder, Path branchLocation) {
@@ -116,25 +130,29 @@ boolean hasTag(Tag tag) {
return tags.contains(tag);
}

void setBranchHash(Branch branch, Hash hash) {
var entry = new ResolvedBranch(branch, hash);
void setBranchHash(Branch branch, String updater, Hash hash) {
var entry = new ResolvedBranch(branch, updater, hash);

branchStorage.put(entry);
var newBranchHashes = currentBranchHashes();

// Sanity check
if (branches != null) {
for (var existingBranch : branches.keySet()) {
if (branchHashes != null) {
for (var existingBranch : branchHashes.keySet()) {
if (!newBranchHashes.containsKey(existingBranch)) {
throw new RuntimeException("Hash information for branch '" + existingBranch + "' is missing");
}
}
}
branches = newBranchHashes;
branchHashes = newBranchHashes;
}

Optional<Hash> branchHash(Branch branch, String updater) {
var entry = branchHashes.get(branch.toString() + " " + updater);
return Optional.ofNullable(entry);
}

Optional<Hash> branchHash(Branch branch) {
var hash = branches.get(branch);
return Optional.ofNullable(hash);
boolean isEmpty() {
return branchHashes.isEmpty();
}
}
@@ -84,17 +84,42 @@ void branchesRetained(TestInfo testInfo) throws IOException {

var history = createHistory(repository, ref);

history.setBranchHash(new Branch("1"), new Hash("a"));
history.setBranchHash(new Branch("2"), new Hash("b"));
history.setBranchHash(new Branch("1"), new Hash("c"));
history.setBranchHash(new Branch("1"), "a", new Hash("a"));
history.setBranchHash(new Branch("2"), "a", new Hash("b"));
history.setBranchHash(new Branch("1"), "a", new Hash("c"));

assertEquals(new Hash("c"), history.branchHash(new Branch("1")).orElseThrow());
assertEquals(new Hash("b"), history.branchHash(new Branch("2")).orElseThrow());
assertEquals(new Hash("c"), history.branchHash(new Branch("1"), "a").orElseThrow());
assertEquals(new Hash("b"), history.branchHash(new Branch("2"), "a").orElseThrow());

var newHistory = createHistory(repository, ref);

assertEquals(new Hash("c"), newHistory.branchHash(new Branch("1")).orElseThrow());
assertEquals(new Hash("b"), newHistory.branchHash(new Branch("2")).orElseThrow());
assertEquals(new Hash("c"), newHistory.branchHash(new Branch("1"), "a").orElseThrow());
assertEquals(new Hash("b"), newHistory.branchHash(new Branch("2"), "a").orElseThrow());
}
}

@Test
void branchesSeparateUpdaters(TestInfo testInfo) throws IOException {
try (var credentials = new HostCredentials(testInfo)) {
var repository = credentials.getHostedRepository();
var ref = resetHostedRepository(repository);

var history = createHistory(repository, ref);

history.setBranchHash(new Branch("1"), "a", new Hash("a"));
history.setBranchHash(new Branch("2"), "a", new Hash("b"));
history.setBranchHash(new Branch("1"), "b", new Hash("c"));
history.setBranchHash(new Branch("2"), "a", new Hash("d"));

assertEquals(new Hash("a"), history.branchHash(new Branch("1"), "a").orElseThrow());
assertEquals(new Hash("d"), history.branchHash(new Branch("2"), "a").orElseThrow());
assertEquals(new Hash("c"), history.branchHash(new Branch("1"), "b").orElseThrow());

var newHistory = createHistory(repository, ref);

assertEquals(new Hash("a"), newHistory.branchHash(new Branch("1"), "a").orElseThrow());
assertEquals(new Hash("d"), newHistory.branchHash(new Branch("2"), "a").orElseThrow());
assertEquals(new Hash("c"), newHistory.branchHash(new Branch("1"), "b").orElseThrow());
}
}

0 comments on commit 286ccf8

Please sign in to comment.