Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

301: Keep track of branch progress per notifier #509

Closed
wants to merge 2 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
@@ -356,6 +356,11 @@ public boolean isIdempotent() {
return true;
}

@Override
public String name() {
return "issue";
}

@Override
public void handleNewIssue(PullRequest pr, org.openjdk.skara.vcs.openjdk.Issue issue) {
var realIssue = issueProject.issue(issue.id());
@@ -111,4 +111,9 @@ public void handleNewBranch(HostedRepository repository, Repository localReposit
public boolean isIdempotent() {
return false;
}

@Override
public String name() {
return "json";
}
}
@@ -384,4 +384,9 @@ public void handleNewBranch(HostedRepository repository, Repository localReposit
public boolean isIdempotent() {
return false;
}

@Override
public String name() {
return "ml";
}
}
@@ -34,4 +34,5 @@ public interface RepositoryUpdateConsumer {
void handleTagCommit(HostedRepository repository, Repository localRepository, Commit commit, Tag tag, Tag.Annotated annotation);
void handleNewBranch(HostedRepository repository, Repository localRepository, List<Commit> commits, Branch parent, Branch branch);
boolean isIdempotent();
String name();
}
@@ -54,7 +54,7 @@ public class RepositoryWorkItem implements WorkItem {
this.updaters = updaters;
}

private void handleNewRef(Repository localRepo, Reference ref, Collection<Reference> allRefs, boolean runOnlyIdempotent) {
private void handleNewRef(Repository localRepo, Reference ref, Collection<Reference> allRefs, RepositoryUpdateConsumer updater) {
// Figure out the best parent ref
var candidates = new HashSet<>(allRefs);
candidates.remove(ref);
@@ -77,55 +77,68 @@ private void handleNewRef(Repository localRepo, Reference ref, Collection<Refere
throw new RuntimeException("Excessive amount of unique commits on new branch " + ref.name() +
" detected (" + bestParent.getValue().size() + ") - skipping notifications");
}
List<Commit> bestParentCommits;
try {
var bestParentCommits = localRepo.commits(bestParent.getKey().hash().hex() + ".." + ref.hash(), true);
for (var updater : updaters) {
if (updater.isIdempotent() != runOnlyIdempotent) {
continue;
}
var branch = new Branch(ref.name());
var parent = new Branch(bestParent.getKey().name());
updater.handleNewBranch(repository, localRepo, bestParentCommits.asList(), parent, branch);
}
bestParentCommits = localRepo.commits(bestParent.getKey().hash().hex() + ".." + ref.hash(), true).asList();
} catch (IOException e) {
e.printStackTrace();
throw new UncheckedIOException(e);
}
var branch = new Branch(ref.name());
var parent = new Branch(bestParent.getKey().name());
updater.handleNewBranch(repository, localRepo, bestParentCommits, parent, branch);
}

private void handleUpdatedRef(Repository localRepo, Reference ref, List<Commit> commits, boolean runOnlyIdempotent) {
for (var updater : updaters) {
if (updater.isIdempotent() != runOnlyIdempotent) {
continue;
}
var branch = new Branch(ref.name());
updater.handleCommits(repository, localRepo, commits, branch);
}
private void handleUpdatedRef(Repository localRepo, Reference ref, List<Commit> commits, RepositoryUpdateConsumer updater) {
var branch = new Branch(ref.name());
updater.handleCommits(repository, localRepo, commits, branch);
}

private void handleRef(Repository localRepo, UpdateHistory history, Reference ref, Collection<Reference> allRefs) throws IOException {
private List<RuntimeException> handleRef(Repository localRepo, UpdateHistory history, Reference ref, Collection<Reference> allRefs) throws IOException {
var errors = new ArrayList<RuntimeException>();
var branch = new Branch(ref.name());
var lastHash = history.branchHash(branch);
if (lastHash.isEmpty()) {
log.warning("No previous history found for branch '" + branch + "' - resetting mark");
handleNewRef(localRepo, ref, allRefs, true);
history.setBranchHash(branch, ref.hash());
handleNewRef(localRepo, ref, allRefs, false);
} else {
var commitMetadata = localRepo.commitMetadata(lastHash.get() + ".." + ref.hash());
if (commitMetadata.size() == 0) {
return;
}
if (commitMetadata.size() > 1000) {
history.setBranchHash(branch, ref.hash());
throw new RuntimeException("Excessive amount of new commits on branch " + branch.name() +
" detected (" + commitMetadata.size() + ") - skipping notifications");
}
for (var updater : updaters) {
var lastHash = history.branchHash(branch, updater.name());
if (lastHash.isEmpty()) {
log.warning("No previous history found for branch '" + branch + "' and updater '" + updater.name() + " - resetting mark");
if (!updater.isIdempotent()) {
history.setBranchHash(branch, updater.name(), ref.hash());
}
try {
handleNewRef(localRepo, ref, allRefs, updater);
if (updater.isIdempotent()) {
history.setBranchHash(branch, updater.name(), ref.hash());
}
} catch (RuntimeException e) {
errors.add(e);
}
} else {
var commitMetadata = localRepo.commitMetadata(lastHash.get() + ".." + ref.hash());
if (commitMetadata.size() == 0) {
continue;
}
if (commitMetadata.size() > 1000) {
history.setBranchHash(branch, updater.name(), ref.hash());
errors.add(new RuntimeException("Excessive amount of new commits on branch " + branch.name() +
" detected (" + commitMetadata.size() + ") for updater '" +
updater.name() + "' - skipping notifications"));
continue;
}

var commits = localRepo.commits(lastHash.get() + ".." + ref.hash(), true).asList();
handleUpdatedRef(localRepo, ref, commits, true);
history.setBranchHash(branch, ref.hash());
handleUpdatedRef(localRepo, ref, commits, false);
var commits = localRepo.commits(lastHash.get() + ".." + ref.hash(), true).asList();
if (!updater.isIdempotent()) {
history.setBranchHash(branch, updater.name(), ref.hash());
}
try {
handleUpdatedRef(localRepo, ref, commits, updater);
if (updater.isIdempotent()) {
history.setBranchHash(branch, updater.name(), ref.hash());
}
} catch (RuntimeException e) {
errors.add(e);
}
}
}
return errors;
}

private Optional<OpenJDKTag> existingPrevious(OpenJDKTag tag, Set<OpenJDKTag> allJdkTags) {
@@ -276,17 +289,23 @@ public void run(Path scratchPath) {
var history = UpdateHistory.create(tagStorageBuilder, historyPath.resolve("tags"), branchStorageBuilder, historyPath.resolve("branches"));
handleTags(localRepo, history);

boolean hasBranchHistory = knownRefs.stream()
.map(ref -> history.branchHash(new Branch(ref.name())))
.anyMatch(Optional::isPresent);
boolean hasBranchHistory = !history.isEmpty();
var errors = new ArrayList<RuntimeException>();
for (var ref : knownRefs) {
if (!hasBranchHistory) {
log.warning("No previous history found for any branch - resetting mark for '" + ref.name() + "'");
history.setBranchHash(new Branch(ref.name()), ref.hash());
log.warning("No previous history found for any branch - resetting mark for '" + ref.name());
for (var updater : updaters) {
log.info("Resetting mark for branch '" + ref.name() + "' for updater '" + updater.name() + "'");
history.setBranchHash(new Branch(ref.name()), updater.name(), ref.hash());
}
} else {
handleRef(localRepo, history, ref, knownRefs);
errors.addAll(handleRef(localRepo, history, ref, knownRefs));
}
}
if (!errors.isEmpty()) {
errors.forEach(error -> log.throwing("RepositoryWorkItem", "run", error));
throw new RuntimeException("Errors detected during ref updating");
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
@@ -28,24 +28,25 @@

class ResolvedBranch {
private final Branch branch;
private final String updater;
private final Hash hash;

ResolvedBranch(Branch branch, Hash hash) {
ResolvedBranch(Branch branch, String updater, Hash hash) {
this.branch = branch;
this.updater = updater;
this.hash = hash;
}

public Branch branch() {
return branch;
}

public Hash hash() {
return hash;
public String updater() {
return updater;
}

@Override
public String toString() {
return branch.name() + ":" + hash().hex();
public Hash hash() {
return hash;
}

@Override
@@ -57,12 +58,11 @@ public boolean equals(Object o) {
return false;
}
ResolvedBranch that = (ResolvedBranch) o;
return Objects.equals(branch, that.branch) &&
Objects.equals(hash, that.hash);
return branch.equals(that.branch) && updater.equals(that.updater) && hash.equals(that.hash);
}

@Override
public int hashCode() {
return Objects.hash(branch, hash);
return Objects.hash(branch, updater, hash);
}
}
@@ -27,30 +27,44 @@

import java.nio.file.Path;
import java.util.*;
import java.util.function.Function;
import java.util.stream.*;

class UpdateHistory {

private final Storage<Tag> tagStorage;
private final Storage<ResolvedBranch> branchStorage;

private Map<Branch, Hash> branches;
private Map<String, Hash> branchHashes;
private Set<Tag> tags;

private List<ResolvedBranch> parseSerializedEntry(String entry) {
var parts = entry.split(" ");
if (parts.length == 2) {
// Transform legacy entry
var issueEntry = new ResolvedBranch(new Branch(parts[0]), "issue", new Hash(parts[1]));
var mlEntry = new ResolvedBranch(new Branch(parts[0]), "ml", new Hash(parts[1]));
return List.of(issueEntry, mlEntry);
}
return List.of(new ResolvedBranch(new Branch(parts[0]), parts[1], new Hash(parts[2])));
}

private Set<ResolvedBranch> loadBranches(String current) {
return current.lines()
.map(line -> line.split(" "))
.map(entry -> new ResolvedBranch(new Branch(entry[0]), new Hash(entry[1])))
.flatMap(line -> parseSerializedEntry(line).stream())
.collect(Collectors.toSet());
}

private String serializeEntry(ResolvedBranch entry) {
return entry.branch().toString() + " " + entry.updater() + " " + entry.hash().toString();
}

private String serializeBranches(Collection<ResolvedBranch> added, Set<ResolvedBranch> existing) {
var updatedBranches = existing.stream()
.collect(Collectors.toMap(ResolvedBranch::branch,
ResolvedBranch::hash));
added.forEach(a -> updatedBranches.put(a.branch(), a.hash()));
return updatedBranches.entrySet().stream()
.map(entry -> entry.getKey().toString() + " " + entry.getValue().toString())
.collect(Collectors.toMap(entry -> entry.branch().toString() + ":" + entry.updater(),
Function.identity()));
added.forEach(a -> updatedBranches.put(a.branch().toString() + ":" + a.updater(), a));
return updatedBranches.values().stream()
.map(this::serializeEntry)
.sorted()
.collect(Collectors.joining("\n"));
}
@@ -73,9 +87,9 @@ private Set<Tag> currentTags() {
return tagStorage.current();
}

private Map<Branch, Hash> currentBranchHashes() {
private Map<String, Hash> currentBranchHashes() {
return branchStorage.current().stream()
.collect(Collectors.toMap(ResolvedBranch::branch, ResolvedBranch::hash));
.collect(Collectors.toMap(rb -> rb.branch().toString() + " " + rb.updater(), ResolvedBranch::hash));
}

private UpdateHistory(StorageBuilder<Tag> tagStorageBuilder, Path tagLocation, StorageBuilder<ResolvedBranch> branchStorageBuilder, Path branchLocation) {
@@ -90,7 +104,7 @@ private UpdateHistory(StorageBuilder<Tag> tagStorageBuilder, Path tagLocation, S
.materialize(branchLocation);

tags = currentTags();
branches = currentBranchHashes();
branchHashes = currentBranchHashes();
}

static UpdateHistory create(StorageBuilder<Tag> tagStorageBuilder, Path tagLocation, StorageBuilder<ResolvedBranch> branchStorageBuilder, Path branchLocation) {
@@ -116,25 +130,29 @@ boolean hasTag(Tag tag) {
return tags.contains(tag);
}

void setBranchHash(Branch branch, Hash hash) {
var entry = new ResolvedBranch(branch, hash);
void setBranchHash(Branch branch, String updater, Hash hash) {
var entry = new ResolvedBranch(branch, updater, hash);

branchStorage.put(entry);
var newBranchHashes = currentBranchHashes();

// Sanity check
if (branches != null) {
for (var existingBranch : branches.keySet()) {
if (branchHashes != null) {
for (var existingBranch : branchHashes.keySet()) {
if (!newBranchHashes.containsKey(existingBranch)) {
throw new RuntimeException("Hash information for branch '" + existingBranch + "' is missing");
}
}
}
branches = newBranchHashes;
branchHashes = newBranchHashes;
}

Optional<Hash> branchHash(Branch branch, String updater) {
var entry = branchHashes.get(branch.toString() + " " + updater);
return Optional.ofNullable(entry);
}

Optional<Hash> branchHash(Branch branch) {
var hash = branches.get(branch);
return Optional.ofNullable(hash);
boolean isEmpty() {
return branchHashes.isEmpty();
}
}
@@ -84,17 +84,42 @@ void branchesRetained(TestInfo testInfo) throws IOException {

var history = createHistory(repository, ref);

history.setBranchHash(new Branch("1"), new Hash("a"));
history.setBranchHash(new Branch("2"), new Hash("b"));
history.setBranchHash(new Branch("1"), new Hash("c"));
history.setBranchHash(new Branch("1"), "a", new Hash("a"));
history.setBranchHash(new Branch("2"), "a", new Hash("b"));
history.setBranchHash(new Branch("1"), "a", new Hash("c"));

assertEquals(new Hash("c"), history.branchHash(new Branch("1")).orElseThrow());
assertEquals(new Hash("b"), history.branchHash(new Branch("2")).orElseThrow());
assertEquals(new Hash("c"), history.branchHash(new Branch("1"), "a").orElseThrow());
assertEquals(new Hash("b"), history.branchHash(new Branch("2"), "a").orElseThrow());

var newHistory = createHistory(repository, ref);

assertEquals(new Hash("c"), newHistory.branchHash(new Branch("1")).orElseThrow());
assertEquals(new Hash("b"), newHistory.branchHash(new Branch("2")).orElseThrow());
assertEquals(new Hash("c"), newHistory.branchHash(new Branch("1"), "a").orElseThrow());
assertEquals(new Hash("b"), newHistory.branchHash(new Branch("2"), "a").orElseThrow());
}
}

@Test
void branchesSeparateUpdaters(TestInfo testInfo) throws IOException {
try (var credentials = new HostCredentials(testInfo)) {
var repository = credentials.getHostedRepository();
var ref = resetHostedRepository(repository);

var history = createHistory(repository, ref);

history.setBranchHash(new Branch("1"), "a", new Hash("a"));
history.setBranchHash(new Branch("2"), "a", new Hash("b"));
history.setBranchHash(new Branch("1"), "b", new Hash("c"));
history.setBranchHash(new Branch("2"), "a", new Hash("d"));

assertEquals(new Hash("a"), history.branchHash(new Branch("1"), "a").orElseThrow());
assertEquals(new Hash("d"), history.branchHash(new Branch("2"), "a").orElseThrow());
assertEquals(new Hash("c"), history.branchHash(new Branch("1"), "b").orElseThrow());

var newHistory = createHistory(repository, ref);

assertEquals(new Hash("a"), newHistory.branchHash(new Branch("1"), "a").orElseThrow());
assertEquals(new Hash("d"), newHistory.branchHash(new Branch("2"), "a").orElseThrow());
assertEquals(new Hash("c"), newHistory.branchHash(new Branch("1"), "b").orElseThrow());
}
}