Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Show a summary of pre-submit testing results in the PR body (if any) #855

Closed
wants to merge 3 commits into from
Closed
Changes from all commits
Commits
File filter
Filter file types
Beta Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.

Always

Just for now

@@ -31,8 +31,8 @@

import java.io.*;
import java.nio.file.Path;
import java.util.*;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.stream.*;
@@ -370,7 +370,6 @@ private boolean relaxedEquals(String s1, String s2) {
.replaceAll("\\s+", " "));
}


private String getStatusMessage(List<Comment> comments, List<Review> reviews, PullRequestCheckIssueVisitor visitor,
List<String> additionalErrors, List<String> integrationBlockers) {
var progressBody = new StringBuilder();
@@ -390,6 +389,14 @@ private String getStatusMessage(List<Comment> comments, List<Review> reviews, Pu
progressBody.append(warningListToText(allAdditionalErrors));
}

if (pr.sourceRepository().isPresent()) {
var sourceRepo = pr.sourceRepository().get();
var checks = sourceRepo.allChecks(pr.headHash());

var resultSummary = TestResults.summarize(checks);
resultSummary.ifPresent(progressBody::append);
}

if (!integrationBlockers.isEmpty()) {
progressBody.append("\n\n### Integration blocker");
if (integrationBlockers.size() > 1) {
@@ -541,7 +548,7 @@ private String getMergeReadyComment(String commitMessage, List<Review> reviews)
message.append(pr.repository().name());
message.append("/blob/");
message.append(pr.targetRef());
message.append("/CONTRIBUTING.md) for more details.");
message.append("/CONTRIBUTING.md) for details.");
}
} catch (IOException e) {
throw new UncheckedIOException(e);
@@ -0,0 +1,165 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package org.openjdk.skara.bots.pr;

import org.openjdk.skara.forge.*;

import java.time.ZonedDateTime;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;

public class TestResults {
private static String platformFromName(String checkName) {
var checkFlavorStart = checkName.indexOf("(");
if (checkFlavorStart > 0) {
return checkName.substring(0, checkFlavorStart - 1).strip();
} else {
return checkName.strip();
}
}

private static String flavorFromName(String checkName) {
var checkFlavorStart = checkName.indexOf("(");
var checkFlavorEnd = checkName.lastIndexOf(")");
if (checkFlavorStart > 0 && checkFlavorEnd > checkFlavorStart) {
var flavor = checkName.substring(checkFlavorStart + 1, checkFlavorEnd).strip().toLowerCase();
for (int i = 1; i < 10; ++i) {
if (flavor.contains("tier" + i)) {
return "Test (tier" + i + ")";
}
}
if (flavor.contains("build")) {
return "Build";
}
}
// Fallback value
return "Build / test";
}

private static boolean ignoredCheck(String checkName) {
var lcName = checkName.toLowerCase();
return lcName.contains("jcheck") || lcName.contains("prerequisites") || lcName.contains("post-process");
}

static Optional<String> summarize(List<Check> checks) {
// Retain only the latest when there are multiple checks with the same name
var latestChecks = checks.stream()
.filter(check -> !ignoredCheck(check.name()))
.sorted(Comparator.comparing(Check::startedAt, ZonedDateTime::compareTo))
.collect(Collectors.toMap(Check::name, Function.identity(), (a, b) -> b, LinkedHashMap::new));
if (latestChecks.isEmpty()) {
return Optional.empty();
}

var platforms = latestChecks.values().stream()
.map(check -> platformFromName(check.name()))
.collect(Collectors.toCollection(TreeSet::new));
var flavors = latestChecks.values().stream()
.map(check -> flavorFromName(check.name()))
.collect(Collectors.toCollection(TreeSet::new));
if (platforms.isEmpty() || flavors.isEmpty()) {
return Optional.empty();
}

var platformFlavors = latestChecks.values().stream()
.collect(Collectors.groupingBy(check -> platformFromName(check.name()))).entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey,
entry -> entry.getValue().stream()
.collect(Collectors.groupingBy(check -> flavorFromName(check.name())))));

var resultsBody = new StringBuilder();
resultsBody.append("\n\n### Successful test tasks\n\n");
resultsBody.append("| |");
platforms.forEach(platform -> resultsBody.append(" ").append(platform).append(" |"));
resultsBody.append("\n| --- |");
platforms.forEach(platform -> resultsBody.append(" ----- |"));
for (var flavor : flavors) {
resultsBody.append("\n| ").append(flavor).append(" |");
for (var platform : platforms) {
var platformChecks = platformFlavors.get(platform);
var flavorChecks = platformChecks.get(flavor);
if (flavorChecks != null) {
int failureCount = 0;
int pendingCount = 0;
int successCount = 0;
for (var check : flavorChecks) {
switch (check.status()) {
case IN_PROGRESS:
pendingCount++;
break;
case FAILURE:
failureCount++;
break;
case SUCCESS:
successCount++;
break;
}
}
int total = failureCount + pendingCount + successCount;
if (failureCount > 0) {
resultsBody.append("");
resultsBody.append(" (").append(failureCount).append("/").append(total).append(" failed) |");
} else if (pendingCount > 0) {
resultsBody.append("");
resultsBody.append(" (").append(pendingCount).append("/").append(total).append(" in progress) |");
} else {
resultsBody.append(" ✔️");
resultsBody.append(" (").append(successCount).append("/").append(total).append(" passed) |");
}

} else {
resultsBody.append(" | ");
}
}
}

var failedChecks = latestChecks.values().stream()
.filter(check -> check.status() == CheckStatus.FAILURE)
.sorted(Comparator.comparing(Check::name))
.collect(Collectors.toList());
if (!failedChecks.isEmpty()) {
resultsBody.append("\n\n**Failed test task");
if (failedChecks.size() > 1) {
resultsBody.append("s");
}
resultsBody.append("**");
for (var failedCheck : failedChecks) {
resultsBody.append("\n- ");
if (failedCheck.details().isPresent()) {
resultsBody.append("[");
resultsBody.append(failedCheck.name());
resultsBody.append("](");
resultsBody.append(failedCheck.details().get().toString());
resultsBody.append(")");
} else {
resultsBody.append("`");
resultsBody.append(failedCheck.name());
resultsBody.append("`");
}
}
}

return Optional.of(resultsBody.toString());
}
}
@@ -28,6 +28,7 @@
import org.openjdk.skara.test.*;

import java.io.IOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.nio.file.attribute.PosixFilePermission;
@@ -1681,4 +1682,53 @@ void differentAuthors(TestInfo testInfo) throws IOException {
assertEquals(numComments, pr.comments().size());
}
}

@Test
void preSubmitInSummary(TestInfo testInfo) throws IOException {
try (var credentials = new HostCredentials(testInfo);
var tempFolder = new TemporaryDirectory()) {
var author = credentials.getHostedRepository();
var reviewer = credentials.getHostedRepository();
var issues = credentials.getIssueProject();

var censusBuilder = credentials.getCensusBuilder()
.addAuthor(author.forge().currentUser().id())
.addReviewer(reviewer.forge().currentUser().id());
var checkBot = PullRequestBot.newBuilder().repo(author).censusRepo(censusBuilder.build()).issueProject(issues).build();

// Populate the projects repository
var localRepo = CheckableRepository.init(tempFolder.path(), author.repositoryType(), Path.of("appendable.txt"),
Set.of("issues"), null);
var masterHash = localRepo.resolve("master").orElseThrow();
localRepo.push(masterHash, author.url(), "master", true);

// Make a draft PR where we can add some checks
var editHash = CheckableRepository.appendAndCommit(localRepo);
localRepo.push(editHash, author.url(), "preedit", true);
var draftPr = credentials.createPullRequest(author, "master", "preedit", "This is a pull request", true);
var check1 = CheckBuilder.create("ps1", editHash).title("PS1");
draftPr.createCheck(check1.build());
draftPr.updateCheck(check1.complete(true).build());
var check2 = CheckBuilder.create("ps2", editHash).title("PS2");
draftPr.createCheck(check2.build());
draftPr.updateCheck(check2.complete(false).build());
var check3 = CheckBuilder.create("ps3", editHash).title("PS3");
draftPr.createCheck(check3.build());
draftPr.updateCheck(check3.details(URI.create("https://www.example.com")).complete(false).build());

// Now make an actual PR
localRepo.push(editHash, author.url(), "edit", true);
var pr = credentials.createPullRequest(author, "master", "edit", "This is a pull request");

// Check the status
TestBotRunner.runPeriodicItems(checkBot);

// The body should contain the issue title
assertTrue(pr.body().contains("Successful test task"));
assertTrue(pr.body().contains("| | ps1 | ps2 | ps3 |"));
assertTrue(pr.body().contains("**Failed test tasks**"));
assertTrue(pr.body().contains("- [ps3](https://www.example.com)"));
assertTrue(pr.body().contains("- `ps2`"));
}
}
}
ProTip! Use n and p to navigate between commits in a pull request.