licenseUrlReplacements;
+ /**
+ * A map that helps to select names for the local files where the downloaded licenses are stored.
+ *
+ * Keys in the map are local file names. These files will be created under {@link #licensesOutputDirectory}.
+ *
+ * Values are white space ({@code " \t\n\r"}) separated lists of regular expressions that will be used to match
+ * license URLs. The regular expressions are compiled using {@link Pattern#CASE_INSENSITIVE}. Note that various
+ * characters that commonly occur in URLs have special meanings in regular extensions. Therefore, consider using
+ * regex quoting as described in {@link Pattern}.
+ *
+ * In addition to URL patterns, the list must also contain the sha1 checksum of the expected content. This is to
+ * ensure that URLs returning distinct licenses cannot be saved under the same local path. Note that strict checking
+ * of the checksum happens only when {@link #forceDownload} is {@code true}. Otherwise the mojo assumes the URL
+ * -> local name mapping is correct and downloads from the URL only if the local file does not exist.
+ *
+ * Using {@link #licenseUrlFileNames} for URLs returning volatile content is not a good idea.
+ *
An example:
+ *
+ * {@code
+ *
+ *
+ * sha1:81ffbd1712afe8cdf138b570c0fc9934742c33c1
+ * https?://(www\.)?antlr\.org/license\.html
+ *
+ *
+ * sha1:534a3fc9ae1076409bb00d01127dbba1e2620e92
+ * \Qhttps://raw.githubusercontent.com/javaee/activation/master/LICENSE.txt\E
+ *
+ *
+ * }
+ *
+ *
+ * Relationship to other parameters:
+ *
+ * - {@link #licenseUrlReplacements} are applied before {@link #licenseUrlFileNames}
+ * - {@link #licenseUrlFileNames} have higher precedence than {@code } elements in
+ * {@link #licensesConfigFile}
+ * - {@link #licenseUrlFileNames} are ignored when {@link #organizeLicensesByDependencies} is {@code true}
+ *
+ *
+ * @since 1.18
+ */
+ @Parameter
+ private Map licenseUrlFileNames;
+
// ----------------------------------------------------------------------
// Plexus Components
// ----------------------------------------------------------------------
@@ -369,13 +417,13 @@ public abstract class AbstractDownloadLicensesMojo
// ----------------------------------------------------------------------
// Private Fields
// ----------------------------------------------------------------------
-
+ private PreferredFileNames preferredFileNames;
/**
* A map from the license URLs to file names (without path) where the
* licenses were downloaded. This helps the plugin to avoid downloading
* the same license multiple times.
*/
- private Map downloadedLicenseURLs = new HashMap<>();
+ private Cache cache;
/**
* Proxy Login/Password encoded(only if usgin a proxy with authentication).
@@ -436,6 +484,8 @@ public void execute()
}
this.errorRemedy = getEffectiveErrorRemedy( this.quiet, this.errorRemedy );
+ this.preferredFileNames = PreferredFileNames.build( licensesOutputDirectory, licenseUrlFileNames, getLog() );
+ this.cache = new Cache( licenseUrlFileNames != null && !licenseUrlFileNames.isEmpty() );
initDirectories();
@@ -479,14 +529,27 @@ public void execute()
{
depProject = createDependencyProject( project );
}
- if ( !offline )
+ depProjectLicenses.add( depProject );
+ }
+ if ( !offline )
+ {
+ /* First save the matching URLs into the cache */
+ for ( ProjectLicenseInfo depProject : depProjectLicenses )
+ {
+ downloadLicenses( licenseDownloader, depProject, true );
+ }
+ getLog().debug( "Finished populating cache" );
+ /*
+ * Then attempt to download the rest of the URLs using the available cache entries to select local
+ * file names based on file content sha1
+ */
+ for ( ProjectLicenseInfo depProject : depProjectLicenses )
{
- downloadLicenses( licenseDownloader, depProject );
+ downloadLicenses( licenseDownloader, depProject, false );
}
- depProjectLicenses.add( depProject );
}
}
- catch ( Exception e )
+ catch ( IOException e )
{
throw new RuntimeException( e );
}
@@ -545,8 +608,12 @@ public void execute()
getLog().warn( "There were " + downloadErrorCount + " download errors - check the warnings above" );
break;
case xmlOutput:
- throw new MojoFailureException( "There were " + downloadErrorCount + " download errors - check "
- + licensesErrorsFile.getAbsolutePath() );
+ if ( downloadErrorCount > 0 )
+ {
+ throw new MojoFailureException( "There were " + downloadErrorCount + " download errors - check "
+ + licensesErrorsFile.getAbsolutePath() );
+ }
+ break;
default:
throw new IllegalStateException( "Unexpected value of " + ErrorRemedy.class.getName() + ": "
+ errorRemedy );
@@ -789,14 +856,30 @@ private void loadLicenseInfo( Map configuredDepLicen
{
for ( ProjectLicense license : dep.getLicenses() )
{
- final String fileName = license.getFile();
- if ( fileName != null )
+ final String url = license.getUrl();
+ if ( url != null )
{
- final File licenseFile = new File( licensesOutputDirectory, fileName );
- if ( licenseFile.exists() )
+ final String licenseUrl = rewriteLicenseUrlIfNecessary( url );
+ final FileNameEntry fileNameEntry =
+ getLicenseFileName( dep, licenseUrl, license.getName(), license.getFile() );
+ final File licenseFile = fileNameEntry.getFile();
+ if ( !forceDownload && licenseFile.exists() )
{
+ final String actualSha1 = FileUtil.sha1( licenseFile.toPath() );
+ if ( fileNameEntry.getSha1() != null && !actualSha1.equals( fileNameEntry.getSha1() ) )
+ {
+ throw new MojoFailureException( "Unexpected sha1 checksum for file '"
+ + licenseFile.getAbsolutePath() + "': '" + actualSha1 + "'; expected '"
+ + fileNameEntry.getSha1() + "'. You may want to (a) re-run the current mojo"
+ + " with -Dlicense.forceDownload=true or (b) change the expected sha1 in"
+ + " the licenseUrlFileNames entry '"
+ + fileNameEntry.getFile().getName() + "' or (c) split the entry so that"
+ + " its URLs return content with different sha1 sums." );
+ }
// Save the URL so we don't download it again
- downloadedLicenseURLs.put( license.getUrl(), licenseFile );
+ cache.put( license.getUrl(), LicenseDownloadResult.success( licenseFile,
+ actualSha1,
+ fileNameEntry.isPreferred() ) );
}
}
}
@@ -850,18 +933,25 @@ private ProjectLicenseInfo createDependencyProject( MavenProject depMavenProject
* @param depProject the project containing the license
* @param licenseUrl the license url
* @param licenseName the license name
+ * @param string
* @return A filename to be used for the downloaded license file
+ * @throws URISyntaxException
*/
- private String getLicenseFileName( ProjectLicenseInfo depProject, final URL licenseUrl, final String licenseName )
+ private FileNameEntry getLicenseFileName( ProjectLicenseInfo depProject, final String url,
+ final String licenseName, String licenseFileName )
+ throws URISyntaxException
{
- String defaultExtension = ".txt";
+ final URI licenseUrl = new URI( url );
File licenseUrlFile = new File( licenseUrl.getPath() );
- String licenseFileName;
-
if ( organizeLicensesByDependencies )
{
+ if ( licenseFileName != null && !licenseFileName.isEmpty() )
+ {
+ return new FileNameEntry( new File( licensesOutputDirectory, new File( licenseFileName ).getName() ),
+ false, null );
+ }
licenseFileName = String.format( "%s.%s%s", depProject.getGroupId(), depProject.getArtifactId(),
licenseName != null
? "_" + licenseName
@@ -869,6 +959,19 @@ private String getLicenseFileName( ProjectLicenseInfo depProject, final URL lice
}
else
{
+ final FileNameEntry preferredFileNameEntry = preferredFileNames.getEntryByUrl( url );
+ if ( preferredFileNameEntry != null )
+ {
+ return preferredFileNameEntry;
+ }
+
+ if ( licenseFileName != null && !licenseFileName.isEmpty() )
+ {
+ return new FileNameEntry( new File( licensesOutputDirectory,
+ new File( licenseFileName ).getName() ),
+ false, null );
+ }
+
licenseFileName = licenseUrlFile.getName();
if ( licenseName != null )
@@ -876,13 +979,6 @@ private String getLicenseFileName( ProjectLicenseInfo depProject, final URL lice
licenseFileName = licenseName + " - " + licenseUrlFile.getName();
}
- // Check if the file has a valid file extention
- int extensionIndex = licenseFileName.lastIndexOf( "." );
- if ( extensionIndex == -1 || extensionIndex > ( licenseFileName.length() - 3 ) )
- {
- // This means it isn't a valid file extension, so append the default
- licenseFileName = licenseFileName + defaultExtension;
- }
// Normalize whitespace
licenseFileName = licenseFileName.replaceAll( "\\s+", " " );
}
@@ -890,23 +986,25 @@ private String getLicenseFileName( ProjectLicenseInfo depProject, final URL lice
// lower case and (back)slash removal
licenseFileName = licenseFileName.toLowerCase( Locale.US ).replaceAll( "[\\\\/]+", "_" );
- return licenseFileName;
+ return new FileNameEntry( new File( licensesOutputDirectory, licenseFileName ), false, null );
}
/**
* Download the licenses associated with this project
*
* @param depProject The project which generated the dependency
+ * @param matchingUrlsOnly
* @throws MojoFailureException
*/
- private void downloadLicenses( LicenseDownloader licenseDownloader, ProjectLicenseInfo depProject )
- throws MojoFailureException
+ private void downloadLicenses( LicenseDownloader licenseDownloader, ProjectLicenseInfo depProject,
+ boolean matchingUrlsOnly )
+ throws MojoFailureException
{
getLog().debug( "Downloading license(s) for project " + depProject );
List licenses = depProject.getLicenses();
- if ( depProject.getLicenses() == null || depProject.getLicenses().isEmpty() )
+ if ( matchingUrlsOnly && ( depProject.getLicenses() == null || depProject.getLicenses().isEmpty() ) )
{
handleError( depProject, "No license information available for: " + depProject );
return;
@@ -915,75 +1013,109 @@ private void downloadLicenses( LicenseDownloader licenseDownloader, ProjectLicen
int licenseIndex = 0;
for ( ProjectLicense license : licenses )
{
- if ( license.getUrl() == null )
+ if ( matchingUrlsOnly && license.getUrl() == null )
{
handleError( depProject, "No URL for license at index " + licenseIndex + " in dependency "
- + depProject.toString() );
+ + depProject.toString() );
}
- else
+ else if ( license.getUrl() != null )
{
final String licenseUrl = rewriteLicenseUrlIfNecessary( license.getUrl() );
+
+ final LicenseDownloadResult cachedResult = cache.get( licenseUrl );
try
{
- File licenseOutputFile = downloadedLicenseURLs.get( licenseUrl );
- if ( licenseOutputFile == null )
+ if ( cachedResult != null )
{
- final String licenseFileName;
- if ( license.getFile() != null )
+ if ( cachedResult.isPreferredFileName() == matchingUrlsOnly )
{
- licenseFileName = new File( license.getFile() ).getName();
- }
- else
- {
- licenseFileName = getLicenseFileName( depProject,
- new URL( license.getUrl() ),
- license.getName() );
+ if ( organizeLicensesByDependencies )
+ {
+ final FileNameEntry fileNameEntry =
+ getLicenseFileName( depProject, licenseUrl, license.getName(), license.getFile() );
+ final File cachedFile = cachedResult.getFile();
+ final LicenseDownloadResult byDepsResult;
+ final File byDepsFile = fileNameEntry.getFile();
+ if ( cachedResult.isSuccess() && !cachedFile.equals( byDepsFile ) )
+ {
+ Files.copy( cachedFile.toPath(), byDepsFile.toPath() );
+ byDepsResult = cachedResult.withFile( byDepsFile );
+ }
+ else
+ {
+ byDepsResult = cachedResult;
+ }
+ handleResult( licenseUrl, byDepsResult, depProject, license );
+ }
+ else
+ {
+ handleResult( licenseUrl, cachedResult, depProject, license );
+ }
}
- licenseOutputFile = new File( licensesOutputDirectory, licenseFileName );
+ return;
}
-
- if ( !licenseOutputFile.exists() || forceDownload )
+ else
{
- if ( !downloadedLicenseURLs.containsKey( licenseUrl ) || organizeLicensesByDependencies )
- {
- final LicenseDownloadResult result =
- licenseDownloader.downloadLicense( licenseUrl, proxyLoginPasswordEncoded,
- licenseOutputFile, getLog() );
+ /* No cache entry for the current URL */
+ final FileNameEntry fileNameEntry =
+ getLicenseFileName( depProject, licenseUrl, license.getName(), license.getFile() );
- if ( result.isSuccess() )
+ final File licenseOutputFile = fileNameEntry.getFile();
+ if ( matchingUrlsOnly == fileNameEntry.isPreferred() )
+ {
+ if ( !licenseOutputFile.exists() || forceDownload )
{
- licenseOutputFile = result.getFile();
- downloadedLicenseURLs.put( licenseUrl, licenseOutputFile );
+ LicenseDownloadResult result =
+ licenseDownloader.downloadLicense( licenseUrl, proxyLoginPasswordEncoded,
+ fileNameEntry, getLog() );
+ if ( !organizeLicensesByDependencies && result.isSuccess() )
+ {
+ /* check if we can re-use an existing file that has the same content */
+ final String name = preferredFileNames.getFileNameBySha1( result.getSha1() );
+ if ( name != null )
+ {
+ final File oldFile = result.getFile();
+ if ( !oldFile.getName().equals( name ) )
+ {
+ final File newFile = new File( licensesOutputDirectory, name );
+ result = result.withFile( newFile );
+ oldFile.delete();
+ }
+ }
+ }
+ handleResult( licenseUrl, result, depProject, license );
+ cache.put( licenseUrl, result );
}
- else
+ else if ( licenseOutputFile.exists() )
{
- handleError( depProject, result.getErrorMessage() );
+ final LicenseDownloadResult result =
+ LicenseDownloadResult.success( licenseOutputFile,
+ FileUtil.sha1( licenseOutputFile.toPath() ),
+ fileNameEntry.isPreferred() );
+ handleResult( licenseUrl, result, depProject, license );
+ cache.put( licenseUrl, result );
}
-
}
}
-
- if ( licenseOutputFile != null )
- {
- license.setFile( licenseOutputFile.getName() );
- }
-
}
catch ( URISyntaxException e )
{
handleError( depProject, "POM for dependency " + depProject.toString()
+ " has an invalid license URL: " + licenseUrl );
+ getLog().debug( e );
}
catch ( FileNotFoundException e )
{
handleError( depProject, "POM for dependency " + depProject.toString()
- + " has a license URL that returns file not found: " + licenseUrl );
+ + " has a license URL that returns file not found: " + licenseUrl );
+ getLog().debug( e );
}
catch ( IOException e )
{
handleError( depProject, "Unable to retrieve license from URL '" + licenseUrl + "' for dependency '"
+ depProject.toString() + "': " + e.getMessage() );
+ getLog().debug( e );
}
}
licenseIndex++;
@@ -991,6 +1123,20 @@ private void downloadLicenses( LicenseDownloader licenseDownloader, ProjectLicen
}
+ private void handleResult( String licenseUrl, LicenseDownloadResult result, ProjectLicenseInfo depProject,
+ ProjectLicense license )
+ throws MojoFailureException
+ {
+ if ( result.isSuccess() )
+ {
+ license.setFile( result.getFile().getName() );
+ }
+ else
+ {
+ handleError( depProject, result.getErrorMessage() );
+ }
+ }
+
private void handleError( ProjectLicenseInfo depProject, String msg ) throws MojoFailureException
{
switch ( errorRemedy )
@@ -1004,6 +1150,7 @@ private void handleError( ProjectLicenseInfo depProject, String msg ) throws Moj
case failFast:
throw new MojoFailureException( msg );
case xmlOutput:
+ getLog().debug( msg );
depProject.addDownloaderMessage( msg );
break;
default:
@@ -1041,18 +1188,21 @@ private String rewriteLicenseUrlIfNecessary( final String originalLicenseUrl )
*
* @since 1.18
*/
- public static enum ErrorRemedy
+ public enum ErrorRemedy
{
/** All errors are ignored */
ignore,
/** All errors are output to the log as warnings */
warn,
- /** The first encountered error is logged and a {@link MojoFailureException} is
- * thrown */
+ /**
+ * The first encountered error is logged and a {@link MojoFailureException} is thrown
+ */
failFast,
- /** Error messages are added as {@code } to
+ /**
+ * Error messages are added as {@code } to
* {@link AbstractDownloadLicensesMojo#licensesErrorsFile}; in case there are error messages, the build will
- * fail after processing all dependencies. */
+ * fail after processing all dependencies.
+ */
xmlOutput
}
}
diff --git a/src/main/java/org/codehaus/mojo/license/download/Cache.java b/src/main/java/org/codehaus/mojo/license/download/Cache.java
new file mode 100644
index 000000000..b97c57198
--- /dev/null
+++ b/src/main/java/org/codehaus/mojo/license/download/Cache.java
@@ -0,0 +1,105 @@
+package org.codehaus.mojo.license.download;
+
+/*
+ * #%L
+ * License Maven Plugin
+ * %%
+ * Copyright (C) 2018 Codehaus
+ * %%
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU General Lesser Public
+ * License along with this program. If not, see
+ * .
+ * #L%
+ */
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.codehaus.mojo.license.utils.LicenseDownloader.LicenseDownloadResult;
+
+/**
+ * A simple {@link HashMap} based in-memory cache for storing {@link LicenseDownloadResult}s.
+ *
+ * @author Peter Palaga
+ * @since 1.18
+ */
+public class Cache
+{
+ private final Map urlToFile = new HashMap<>();
+
+ private final Map sha1ToFile = new HashMap<>();
+
+ private final boolean enforcingUniqueSha1s;
+
+ public Cache( boolean enforcingUniqueSha1s )
+ {
+ super();
+ this.enforcingUniqueSha1s = enforcingUniqueSha1s;
+ }
+
+ /**
+ * @param url the cache key to seek
+ * @return the {@link LicenseDownloadResult} bound to the given {@code url} or {@code null} if no entry is bound to
+ * the given {@code url}
+ */
+ public LicenseDownloadResult get( String url )
+ {
+ return urlToFile.get( url );
+ }
+
+ /**
+ * Binds the given {@code url} to the give {@link LicenseDownloadResult}. If both {@link #enforcingUniqueSha1s} and
+ * {@link LicenseDownloadResult#isSuccess()} are {@code true} and an entry with the given
+ * {@link LicenseDownloadResult#getSha1()} already exists in {@link #sha1ToFile}, asserts that both the added and
+ * the available entry point at the same file.
+ *
+ * @param url the URL the given {@code entry} comes from
+ * @param entry the result of downloading from {@code url}
+ */
+ public void put( String url, LicenseDownloadResult entry )
+ {
+ if ( entry.isSuccess() )
+ {
+ final String sha1 = entry.getSha1();
+ final File entryFile = entry.getFile();
+ final LicenseDownloadResult existing = sha1ToFile.get( sha1 );
+ if ( existing == null )
+ {
+ sha1ToFile.put( sha1, entry );
+ }
+ else if ( enforcingUniqueSha1s && !existing.getFile().equals( entryFile ) )
+ {
+ final File existingFile = existing.getFile();
+ final StringBuilder sb = new StringBuilder();
+ for ( Entry en : urlToFile.entrySet() )
+ {
+ if ( existingFile.equals( en.getValue().getFile() ) )
+ {
+ if ( sb.length() > 0 )
+ {
+ sb.append( ", " );
+ }
+ sb.append( en.getKey() );
+ }
+ }
+ throw new IllegalStateException( "URL '" + url
+ + "' should belong to licenseUrlFileName having key '" + existingFile.getName()
+ + "' together with URLs '" + sb.toString() + "'" );
+ }
+ }
+ urlToFile.put( url, entry );
+ }
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/codehaus/mojo/license/download/FileNameEntry.java b/src/main/java/org/codehaus/mojo/license/download/FileNameEntry.java
new file mode 100644
index 000000000..aa6278f4a
--- /dev/null
+++ b/src/main/java/org/codehaus/mojo/license/download/FileNameEntry.java
@@ -0,0 +1,64 @@
+package org.codehaus.mojo.license.download;
+
+/*
+ * #%L
+ * License Maven Plugin
+ * %%
+ * Copyright (C) 2018 Codehaus
+ * %%
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU General Lesser Public
+ * License along with this program. If not, see
+ * .
+ * #L%
+ */
+
+import java.io.File;
+
+/**
+ * A triple consiting of a {@link File}, its SHA-1 checksum and a boolean whether {@link #file}'s name comes from
+ * {@link PreferredFileNames} and is thus preferred.
+ *
+ * @author Peter Palaga
+ * @since 1.18
+ */
+public class FileNameEntry
+{
+ public FileNameEntry( File file, boolean preferred, String sha1 )
+ {
+ super();
+ this.file = file;
+ this.preferred = preferred;
+ this.sha1 = sha1;
+ }
+
+ private final File file;
+
+ private final boolean preferred;
+
+ private final String sha1;
+
+ public File getFile()
+ {
+ return file;
+ }
+
+ public boolean isPreferred()
+ {
+ return preferred;
+ }
+
+ public String getSha1()
+ {
+ return sha1;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/codehaus/mojo/license/download/PreferredFileNames.java b/src/main/java/org/codehaus/mojo/license/download/PreferredFileNames.java
new file mode 100644
index 000000000..3fd534f05
--- /dev/null
+++ b/src/main/java/org/codehaus/mojo/license/download/PreferredFileNames.java
@@ -0,0 +1,152 @@
+package org.codehaus.mojo.license.download;
+
+/*
+ * #%L
+ * License Maven Plugin
+ * %%
+ * Copyright (C) 2018 Codehaus
+ * %%
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Lesser Public License for more details.
+ *
+ * You should have received a copy of the GNU General Lesser Public
+ * License along with this program. If not, see
+ * .
+ * #L%
+ */
+
+import java.io.File;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Pattern;
+
+import org.apache.maven.plugin.logging.Log;
+
+/**
+ * An url -> {@link FileNameEntry} mapping.
+ *
+ * @author Peter Palaga
+ * @since 1.18
+ */
+public class PreferredFileNames
+{
+ /**
+ * @param licensesOutputDirectory
+ * @param licenseUrlFileNames
+ * @param log
+ * @return a new {@link PreferredFileNames} built of AbstractDownloadLicensesMojo.licenseUrlFileNames
+ */
+ public static PreferredFileNames build( File licensesOutputDirectory, Map licenseUrlFileNames,
+ Log log )
+ {
+ final Map>> fileNameToUrlPatterns = new LinkedHashMap<>();
+ final Map sha1TofileName = new LinkedHashMap();
+ if ( licenseUrlFileNames != null )
+ {
+ for ( Entry en : licenseUrlFileNames.entrySet() )
+ {
+ final String fileName = en.getKey();
+ if ( fileName != null && !fileName.isEmpty() )
+ {
+ if ( en.getValue() != null )
+ {
+ String[] rawPatters = en.getValue().split( "\\s+" );
+ if ( rawPatters != null )
+ {
+ final List patterns = new ArrayList<>();
+ String sha1 = null;
+ for ( String rawPattern : rawPatters )
+ {
+ if ( rawPattern.startsWith( "sha1:" ) )
+ {
+ if ( sha1 != null )
+ {
+ throw new IllegalStateException( "sha1 defined twice for licenseFileName '"
+ + fileName + "'" );
+ }
+ sha1 = rawPattern.substring( 5 );
+ }
+ else
+ {
+ patterns.add( Pattern.compile( rawPattern, Pattern.CASE_INSENSITIVE ) );
+ }
+ }
+ if ( sha1 == null )
+ {
+ throw new IllegalStateException( "sha1 undefined for licenseFileName '" + fileName
+ + "'. Add 'sha1:' to the list of patterns '" + en.getValue()
+ + "'" );
+ }
+ fileNameToUrlPatterns.put( fileName,
+ new AbstractMap.SimpleImmutableEntry<>(
+ sha1,
+ Collections.unmodifiableList( patterns ) ) );
+ sha1TofileName.put( sha1, fileName );
+ }
+ }
+ }
+ }
+ }
+ return new PreferredFileNames( licensesOutputDirectory, fileNameToUrlPatterns, sha1TofileName, log );
+
+ }
+
+ private final File licensesOutputDirectory;
+ private final Map>> fileNameToUrlPatterns;
+ private final Map sha1ToFileName;
+ private final Log log;
+
+ public PreferredFileNames( File licensesOutputDirectory,
+ Map>> fileNameToUrlPatterns,
+ Map sha1ToFileName, Log log )
+ {
+ super();
+ this.licensesOutputDirectory = licensesOutputDirectory;
+ this.fileNameToUrlPatterns = fileNameToUrlPatterns;
+ this.sha1ToFileName = sha1ToFileName;
+ this.log = log;
+ }
+
+ /**
+ * @param sha1 the checksum to search by
+ * @return a file name bound the given {@code sha1} checksum or {@code null}
+ */
+ public String getFileNameBySha1( String sha1 )
+ {
+ return sha1ToFileName.get( sha1 );
+ }
+
+ /**
+ * @param url the URL to query
+ * @return the preferred {@link FileNameEntry} for the given {@code url} or {@code null}
+ */
+ public FileNameEntry getEntryByUrl( String url )
+ {
+ for ( Entry>> fn : fileNameToUrlPatterns.entrySet() )
+ {
+ for ( Pattern pat : fn.getValue().getValue() )
+ {
+ if ( pat.matcher( url ).matches() )
+ {
+ log.debug( "Using file name '" + fn.getKey() + "' for URL '" + url + "' that matched pattern '"
+ + pat.pattern() + "'" );
+ return new FileNameEntry( new File( licensesOutputDirectory, fn.getKey() ), true,
+ fn.getValue().getKey() );
+ }
+ }
+ }
+ return null;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/codehaus/mojo/license/utils/FileUtil.java b/src/main/java/org/codehaus/mojo/license/utils/FileUtil.java
index 25747af96..02e616490 100644
--- a/src/main/java/org/codehaus/mojo/license/utils/FileUtil.java
+++ b/src/main/java/org/codehaus/mojo/license/utils/FileUtil.java
@@ -22,6 +22,7 @@
* #L%
*/
+import org.apache.commons.codec.binary.Hex;
import org.codehaus.plexus.util.FileUtils;
import org.codehaus.plexus.util.IOUtil;
@@ -38,6 +39,10 @@
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.StringReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -290,4 +295,17 @@ public int compare( File o1, File o2 )
} );
return result;
}
+
+ public static String sha1( Path in ) throws IOException
+ {
+ try
+ {
+ final MessageDigest md = MessageDigest.getInstance( "SHA-1" );
+ return Hex.encodeHexString( md.digest( Files.readAllBytes( in ) ) );
+ }
+ catch ( NoSuchAlgorithmException e )
+ {
+ throw new RuntimeException( e );
+ }
+ }
}
diff --git a/src/main/java/org/codehaus/mojo/license/utils/LicenseDownloader.java b/src/main/java/org/codehaus/mojo/license/utils/LicenseDownloader.java
index 971ccdb2a..6f2fdfdcc 100644
--- a/src/main/java/org/codehaus/mojo/license/utils/LicenseDownloader.java
+++ b/src/main/java/org/codehaus/mojo/license/utils/LicenseDownloader.java
@@ -26,12 +26,15 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import org.apache.commons.codec.binary.Hex;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
@@ -41,7 +44,9 @@
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
+import org.apache.maven.plugin.MojoFailureException;
import org.apache.maven.plugin.logging.Log;
+import org.codehaus.mojo.license.download.FileNameEntry;
/**
* Utilities for downloading remote license files.
@@ -82,20 +87,25 @@ public LicenseDownloader()
* @return the path to the file where the downloaded license file was stored
* @throws IOException
* @throws URISyntaxException
+ * @throws MojoFailureException
*/
- public LicenseDownloadResult downloadLicense( String licenseUrlString, String loginPassword, File outputFile,
- Log log )
- throws IOException, URISyntaxException
+ public LicenseDownloadResult downloadLicense( String licenseUrlString, String loginPassword,
+ FileNameEntry fileNameEntry, Log log )
+ throws IOException, URISyntaxException, MojoFailureException
{
+ final File outputFile = fileNameEntry.getFile();
if ( licenseUrlString == null || licenseUrlString.length() == 0 )
{
- return LicenseDownloadResult.success( outputFile );
+ throw new IllegalArgumentException( "Null URL for file " + outputFile.getPath() );
}
+ log.debug( "Downloading " + licenseUrlString );
+
if ( licenseUrlString.startsWith( "file://" ) )
{
- Files.copy( Paths.get( new URI( licenseUrlString ) ), outputFile.toPath() );
- return LicenseDownloadResult.success( outputFile );
+ Path in = Paths.get( new URI( licenseUrlString ) );
+ Files.copy( in, outputFile.toPath() );
+ return LicenseDownloadResult.success( outputFile, FileUtil.sha1( in ), fileNameEntry.isPreferred() );
}
else
{
@@ -113,16 +123,40 @@ public LicenseDownloadResult downloadLicense( String licenseUrlString, String lo
if ( entity != null )
{
final ContentType contentType = ContentType.get( entity );
- File updatedFile =
- updateFileExtension( outputFile, contentType != null ? contentType.getMimeType() : null );
+
+ File updatedFile = fileNameEntry.isPreferred() ? outputFile
+ : updateFileExtension( outputFile,
+ contentType != null ? contentType.getMimeType() : null );
try ( InputStream in = entity.getContent();
FileOutputStream fos = new FileOutputStream( updatedFile ) )
{
- copyStream( in, fos );
+ final MessageDigest md = MessageDigest.getInstance( "SHA-1" );
+ final byte[] buf = new byte[1024];
+ int len;
+ while ( ( len = in.read( buf ) ) >= 0 )
+ {
+ md.update( buf, 0, len );
+ fos.write( buf, 0, len );
+ }
+ final String actualSha1 = Hex.encodeHexString( md.digest() );
+ final String expectedSha1 = fileNameEntry.getSha1();
+ if ( expectedSha1 != null && !expectedSha1.equals( actualSha1 ) )
+ {
+ throw new MojoFailureException( "URL '" + licenseUrlString
+ + "' returned content with unexpected sha1 '" + actualSha1 + "'; expected '"
+ + expectedSha1 + "'. You may want to (a) re-run the current mojo"
+ + " with -Dlicense.forceDownload=true or (b) change the expected sha1 in"
+ + " the licenseUrlFileNames entry '"
+ + fileNameEntry.getFile().getName() + "' or (c) split the entry so that"
+ + " its URLs return content with different sha1 sums." );
+ }
+ return LicenseDownloadResult.success( updatedFile, actualSha1, fileNameEntry.isPreferred() );
+ }
+ catch ( NoSuchAlgorithmException e )
+ {
+ throw new RuntimeException( e );
}
- return LicenseDownloadResult.success( updatedFile );
-
}
else
{
@@ -132,24 +166,6 @@ public LicenseDownloadResult downloadLicense( String licenseUrlString, String lo
}
}
- /**
- * Copy data from one stream to another.
- *
- * @param inStream
- * @param outStream
- * @throws IOException
- */
- private static void copyStream( InputStream inStream, OutputStream outStream )
- throws IOException
- {
- byte[] buf = new byte[1024];
- int len;
- while ( ( len = inStream.read( buf ) ) > 0 )
- {
- outStream.write( buf, 0, len );
- }
- }
-
private static File updateFileExtension( File outputFile, String mimeType )
{
final String realExtension = getFileExtension( mimeType );
@@ -161,6 +177,13 @@ private static File updateFileExtension( File outputFile, String mimeType )
return new File( outputFile.getAbsolutePath() + realExtension );
}
}
+ /* default extension is .txt */
+ final String name = outputFile.getName();
+ final int periodPos = name.lastIndexOf( '.' );
+ if ( periodPos < 0 || name.length() - periodPos > 5 )
+ {
+ return new File( outputFile.getParent(), name.substring( 0, periodPos ) + ".txt" );
+ }
return outputFile;
}
@@ -191,8 +214,7 @@ private static String getFileExtension( String mimeType )
}
@Override
- public void close()
- throws Exception
+ public void close() throws IOException
{
client.close();
}
@@ -204,27 +226,33 @@ public void close()
*/
public static class LicenseDownloadResult
{
- public static LicenseDownloadResult success( File file )
+ public static LicenseDownloadResult success( File file, String sha1, boolean preferredFileName )
{
- return new LicenseDownloadResult( file, null );
+ return new LicenseDownloadResult( file, sha1, preferredFileName, null );
}
public static LicenseDownloadResult failure( String errorMessage )
{
- return new LicenseDownloadResult( null, errorMessage );
+ return new LicenseDownloadResult( null, null, false, errorMessage );
}
- private LicenseDownloadResult( File file, String errorMessage )
+ private LicenseDownloadResult( File file, String sha1, boolean preferredFileName, String errorMessage )
{
super();
this.file = file;
this.errorMessage = errorMessage;
+ this.sha1 = sha1;
+ this.preferredFileName = preferredFileName;
}
private final File file;
private final String errorMessage;
+ private final String sha1;
+
+ private final boolean preferredFileName;
+
public File getFile()
{
return file;
@@ -239,6 +267,21 @@ public boolean isSuccess()
{
return errorMessage == null;
}
+
+ public boolean isPreferredFileName()
+ {
+ return preferredFileName;
+ }
+
+ public String getSha1()
+ {
+ return sha1;
+ }
+
+ public LicenseDownloadResult withFile( File otherFile )
+ {
+ return new LicenseDownloadResult( otherFile, sha1, preferredFileName, errorMessage );
+ }
}
}