Skip to content

Commit

Permalink
Fix null SPDX license ID and printing of license statistics (#1112)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dwalluck
dwalluck authored Jul 4, 2024
1 parent f361c0c commit fce9f66
Show file tree
Hide file tree
Showing 6 changed files with 85 additions and 81 deletions.
12 changes: 8 additions & 4 deletions core/src/main/java/org/jboss/pnc/build/finder/core/BuildFinder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1409,7 +1409,8 @@ public Map<BuildSystemInteger, KojiBuild> call() throws KojiClientException {
"Added {} unique SPDX licenses to builds: {}",
green(uniqueLicenses.size()),
green(String.join(", ", uniqueLicenses)));
List<KojiBuild> buildsWithLicenses = buildsFoundList.stream()
List<KojiBuild> buildsWithLicenses = allBuilds.values()
.stream()
.filter(
kojiBuild -> kojiBuild.getArchives()
.stream()
Expand All @@ -1422,7 +1423,8 @@ public Map<BuildSystemInteger, KojiBuild> call() throws KojiClientException {
green(numBuilds),
green(Math.round(((double) numBuildsWithLicenses / (double) numBuilds) * 100D)));

List<KojiLocalArchive> archives = buildsFoundList.stream()
List<KojiLocalArchive> archives = allBuilds.values()
.stream()
.flatMap(kojiBuild -> kojiBuild.getArchives().stream())
.collect(Collectors.toUnmodifiableList());
int numArchives = archives.size();
Expand All @@ -1436,11 +1438,13 @@ public Map<BuildSystemInteger, KojiBuild> call() throws KojiClientException {
green(Math.round(((double) numArchivesWithLicenses / (double) numArchives) * 100D)));

if (LOGGER.isWarnEnabled()) {
List<KojiBuild> buildsWithoutLicenses = new ArrayList<>(buildsFoundList);
List<KojiBuild> buildsWithoutLicenses = new ArrayList<>(allBuilds.values());
buildsWithoutLicenses.removeIf(BuildFinderUtils::isBuildZero);
buildsWithoutLicenses.removeAll(buildsWithLicenses);
int numBuildsWithoutLicenses = buildsWithoutLicenses.size();
LOGGER.warn(
"{} builds are missing licenses: {}",
red(numBuilds - numBuildsWithLicenses),
red(numBuildsWithoutLicenses),
red(
buildsWithoutLicenses.stream()
.map(KojiBuild::getBuildInfo)
Expand Down
65 changes: 29 additions & 36 deletions core/src/main/java/org/jboss/pnc/build/finder/core/DistributionAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import static org.jboss.pnc.build.finder.core.LicenseSource.POM_XML;
import static org.jboss.pnc.build.finder.core.LicenseSource.TEXT;
import static org.jboss.pnc.build.finder.core.LicenseUtils.NOASSERTION;
import static org.jboss.pnc.build.finder.core.LicenseUtils.isUrl;
import static org.jboss.pnc.build.finder.core.MavenUtils.getLicenses;
import static org.jboss.pnc.build.finder.core.MavenUtils.isPom;
import static org.jboss.pnc.build.finder.core.MavenUtils.isPomXml;
Expand Down Expand Up @@ -148,15 +149,13 @@ public class DistributionAnalyzer implements Callable<Map<ChecksumType, MultiVal

private DistributionAnalyzerListener listener;

private Map<String, List<String>> mapping;

public DistributionAnalyzer(List<String> inputs, BuildConfig config) {
this(inputs, config, null);
}

public DistributionAnalyzer(List<String> inputs, BuildConfig config, BasicCacheContainer cacheManager) {
try {
mapping = LicenseUtils.loadLicenseMapping();
Map<String, List<String>> mapping = LicenseUtils.loadLicenseMapping();

if (LOGGER.isInfoEnabled()) {
LOGGER.info(
Expand All @@ -165,7 +164,6 @@ public DistributionAnalyzer(List<String> inputs, BuildConfig config, BasicCacheC
green(String.join(", ", mapping.keySet())));
}
} catch (IOException e) {
mapping = Collections.emptyMap();
LOGGER.error("Error loading SPDX license URL mappings: {}", boldRed(getAllErrorMessages(e)));
LOGGER.debug("Error", e);
}
Expand Down Expand Up @@ -679,7 +677,7 @@ private List<LicenseInfo> addLicensesFromJar(FileObject jar, FileObject localFil
private static List<LicenseInfo> addLicenseFromTextFile(FileObject jar, FileObject licenseFile) throws IOException {
String licenseId = LicenseUtils.getMatchingLicense(licenseFile);
LicenseInfo licenseInfo = new LicenseInfo(
licenseId,
null,
jar.getName().getRelativeName(licenseFile.getName()),
LicenseUtils.getCurrentLicenseId(licenseId),
TEXT);
Expand All @@ -691,26 +689,11 @@ private static List<LicenseInfo> addLicensesFromBundleLicense(FileObject fileObj
List<BundleLicense> bundlesLicenses = LicenseUtils.getBundleLicenseFromManifest(fileObject);

for (BundleLicense bundleLicense : bundlesLicenses) {
String name = bundleLicense.getLicenseIdentifier();

if (name == null) {
name = bundleLicense.getDescription();
}

String licenseIdentifier = bundleLicense.getLicenseIdentifier();
String description = bundleLicense.getDescription();
String name = LicenseUtils.getFirstNonBlankString(licenseIdentifier, description);
String url = bundleLicense.getLink();
LicenseInfo licenseInfo = new LicenseInfo(name, url, BUNDLE_LICENSE);
String licenseId = licenseInfo.getSpdxLicenseId();

if (licenseId.equals(NOASSERTION)) {
if (LOGGER.isWarnEnabled()) {
LOGGER.warn(
"Missing mapping for Bundle-License: name: {}, URL: {} for file {}",
red(name),
red(url),
red(fileObject));
}
}

licenses.add(licenseInfo);
}

Expand All @@ -725,22 +708,19 @@ private List<LicenseInfo> addLicensesFromPom(FileObject fileObject, LicenseSourc
List<LicenseInfo> licenseInfos = entry.getValue();

if (licenseInfos.isEmpty()) {
LicenseInfo licenseInfo = new LicenseInfo(source);
licenseInfos.add(licenseInfo);
return Collections.emptyList();
}

if (LOGGER.isDebugEnabled()) {
if (!licenseInfos.isEmpty()) {
LOGGER.debug(
"Found {} SPDX licenses for {}: {}",
licenseInfos.size(),
pomOrJarFile,
String.join(
", ",
licenseInfos.stream()
.map(LicenseInfo::getSpdxLicenseId)
.collect(Collectors.toUnmodifiableSet())));
}
LOGGER.debug(
"Found {} SPDX licenses for {}: {}",
licenseInfos.size(),
pomOrJarFile,
String.join(
", ",
licenseInfos.stream()
.map(LicenseInfo::getSpdxLicenseId)
.collect(Collectors.toUnmodifiableSet())));
}

return licenseInfos;
Expand All @@ -758,6 +738,19 @@ private void putLicenses(String pomOrJarFile, Collection<LicenseInfo> licenseInf
} else {
licensesMap.put(pomOrJarFile, licenseInfos);
}

if (LOGGER.isWarnEnabled()) {
for (LicenseInfo licenseInfo : licenseInfos) {
if (licenseInfo.getSpdxLicenseId().equals(NOASSERTION)) {
String name = licenseInfo.getName();
String url = licenseInfo.getUrl();

if (name != null || isUrl(url)) {
LOGGER.warn("Missing SPDX license mapping for name: {}, URL: {}", red(name), red(url));
}
}
}
}
}

public List<String> getInputs() {
Expand Down
21 changes: 0 additions & 21 deletions core/src/main/java/org/jboss/pnc/build/finder/core/LicenseInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,6 @@
*/
package org.jboss.pnc.build.finder.core;

import static org.jboss.pnc.build.finder.core.LicenseSource.UNKNOWN;
import static org.jboss.pnc.build.finder.core.LicenseUtils.NONE;

import java.util.Objects;

import org.apache.maven.model.License;
Expand All @@ -35,24 +32,6 @@ public class LicenseInfo implements Comparable<LicenseInfo> {

private final LicenseSource source;

public LicenseInfo() {
comments = null;
distribution = null;
name = null;
url = null;
spdxLicenseId = NONE;
source = UNKNOWN;
}

public LicenseInfo(LicenseSource source) {
comments = null;
distribution = null;
this.name = null;
this.url = null;
this.spdxLicenseId = null;
this.source = source;
}

public LicenseInfo(License license, LicenseSource source) {
comments = license.getComments();
distribution = license.getDistribution();
Expand Down
58 changes: 40 additions & 18 deletions core/src/main/java/org/jboss/pnc/build/finder/core/LicenseUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ static String normalizeLicenseUrl(String licenseUrl) {
*/
public static boolean isUrl(String... strings) {
for (String s : strings) {
if (s == null || StringUtils.containsWhitespace(s) || !s.contains(URL_MARKER)) {
if (s == null || !s.contains(URL_MARKER) || StringUtils.containsWhitespace(s)) {
return false;
}
}
Expand All @@ -314,11 +314,7 @@ public static Optional<String> findLicenseMapping(String licenseString) {
}

static Optional<String> findLicenseMapping(Map<String, List<String>> mapping, String licenseString) {
if (licenseString == null) {
return Optional.empty();
}

if (licenseString.isBlank()) {
if (StringUtils.isBlank(licenseString)) {
return Optional.empty();
}

Expand All @@ -338,8 +334,9 @@ static Optional<String> findLicenseMapping(Map<String, List<String>> mapping, St
}
} else {
String normalizedString = StringUtils.normalizeSpace(licenseString);
String normalizedNameOrUrl = StringUtils.normalizeSpace(licenseNameOrUrl);

if (StringUtils.equalsIgnoreCase(licenseNameOrUrl, normalizedString)) {
if (StringUtils.equalsIgnoreCase(normalizedNameOrUrl, normalizedString)) {
return Optional.of(getCurrentLicenseId(licenseId));
}
}
Expand All @@ -349,6 +346,10 @@ static Optional<String> findLicenseMapping(Map<String, List<String>> mapping, St
return Optional.empty();
}

static String getSPDXLicenseId(Map<String, List<String>> mapping, String name, String url) {
return findSPDXLicenseId(mapping, name, url).orElse(NOASSERTION);
}

/**
* Gets the SPDX license identifier using the loaded mappings.
*
Expand All @@ -360,10 +361,10 @@ public static String getSPDXLicenseId(String name, String url) {
return getSPDXLicenseId(MAPPING, name, url);
}

static String getSPDXLicenseId(Map<String, List<String>> mapping, String name, String url) {
Optional<String> licenseMapping = StringUtils.isBlank(url) ? LicenseUtils.findLicenseMapping(mapping, name)
: LicenseUtils.findLicenseMapping(mapping, url);
return licenseMapping.or(() -> LicenseUtils.findMatchingLicense(name, url)).orElse(NOASSERTION);
static Optional<String> findSPDXLicenseId(Map<String, List<String>> mapping, String name, String url) {
Optional<String> optSPDXLicenseId = LicenseUtils.findLicenseMapping(mapping, url)
.or(() -> LicenseUtils.findMatchingLicense(name, url));
return optSPDXLicenseId.or(() -> LicenseUtils.findLicenseMapping(mapping, name));
}

/**
Expand Down Expand Up @@ -479,13 +480,7 @@ static boolean containsWordsInSameOrder(String licenseStringCandidate, String li
return true;
}

/**
* Finds a match for the URL (after normalization) in the SPDX seeAlso URL list.
*
* @param licenseUrl the license URL
* @return the matching SPDX license URL if found, otherwise empty
*/
public static Optional<String> findMatchingLicenseSeeAlso(String licenseUrl) {
static Optional<String> findMatchingLicenseSeeAlso(String licenseUrl) {
if (licenseUrl == null || licenseUrl.isBlank()) {
return Optional.empty();
}
Expand Down Expand Up @@ -513,6 +508,7 @@ public static Optional<String> findMatchingLicenseSeeAlso(String licenseUrl) {
/**
* Finds a match for the given license name and license URL, if any, in the SPDX license list. A match is searched
* for in the following order:
*
* <ol>
* <li>Search for a matching SPDX license identifier</li>
* <li>Search for a match SPDX license name</li>
Expand Down Expand Up @@ -671,4 +667,30 @@ static String getCurrentLicenseId(String licenseId) {
String currentId = DEPRECATED_LICENSE_IDS_MAP.get(licenseId);
return currentId != null ? currentId : licenseId;
}

/**
* Gets the first non-blank string.
*
* @param strings the strings
* @return the first non-blank string. or null
*/
public static String getFirstNonBlankString(String... strings) {
return findFirstNonBlankString(strings).orElse(null);
}

/**
* Finds the first non-blank string.
*
* @param strings the strings
* @return the first non-blank string. or empty.
*/
public static Optional<String> findFirstNonBlankString(String... strings) {
for (String string : strings) {
if (!StringUtils.isBlank(string)) {
return Optional.of(string);
}
}

return Optional.empty();
}
}
6 changes: 5 additions & 1 deletion core/src/main/resources/license-mapping.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@
"https://raw.githubusercontent.com/googleapis/java-cloud-bom/libraries-bom-v25.4.0/LICENSE",
"AL2",
"Apache License",
"src/assemble/EHCACHE-CORE-LICENSE.txt"
"see: http://freemarker.org/LICENSE.txt",
"src/assemble/EHCACHE-CORE-LICENSE.txt",
"The Apache License",
"Version 2.0"
],
"BSD-2-Clause": [
"http://www.opensource.org/licenses/bsd-license.html",
Expand All @@ -22,6 +25,7 @@
"http://jaxen.codehaus.org/license.html",
"http://repository.jboss.org/licenses/edl-1.0.txt",
"http://www.antlr.org/license.html",
"http://www.jcraft.com/jzlib/LICENSE.txt",
"http://www.thaiopensource.com/relaxng/copying.html",
"http://xmlunit.svn.sourceforge.net/viewvc/*checkout*/xmlunit/trunk/xmlunit/LICENSE.txt",
"http://x-stream.github.io/license.html",
Expand Down
4 changes: 3 additions & 1 deletion core/src/test/java/org/jboss/pnc/build/finder/core/LicenseUtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ void testFindMatchingLicenseName() {
@Test
void testGetSPDXLicenseId() {
Map<String, List<String>> mapping = Map.of(NONE, List.of("Public Domain"));
assertThat(LicenseUtils.getSPDXLicenseId(mapping, "Public Domain", null)).isEqualTo(NONE);
assertThat(LicenseUtils.getSPDXLicenseId(mapping, " Public Domain\n", null)).isEqualTo(NONE);
assertThat(LicenseUtils.getSPDXLicenseId(mapping, null, "http://repository.jboss.org/licenses/cc0-1.0.txt"))
.isEqualTo("CC0-1.0");
assertThat(
Expand Down Expand Up @@ -175,6 +175,8 @@ void testFindLicenseMapping() {
.hasValue("LGPL-2.1-only");
assertThat(LicenseUtils.findLicenseMapping("https://projects.eclipse.org/license/secondary-gpl-2.0-cp"))
.hasValue("GPL-2.0-only WITH Classpath-exception-2.0");
assertThat(LicenseUtils.findLicenseMapping("http://www.jcraft.com/jzlib/LICENSE.txt")).hasValue("BSD-3-Clause");
assertThat(LicenseUtils.findLicenseMapping("see: http://freemarker.org/LICENSE.txt")).hasValue("Apache-2.0");
}

@Test
Expand Down

0 comments on commit fce9f66

Please sign in to comment.