From 5e9923cfe505537753844113b54e5381875cf307 Mon Sep 17 00:00:00 2001 From: Peter Palaga Date: Tue, 5 Feb 2019 22:19:46 +0100 Subject: [PATCH 1/2] Fixup #237 Build with errorRemedy = xmlOutput should fail if there are download errors --- .../mojo/license/AbstractDownloadLicensesMojo.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/codehaus/mojo/license/AbstractDownloadLicensesMojo.java b/src/main/java/org/codehaus/mojo/license/AbstractDownloadLicensesMojo.java index 56b6d024e..0e2a39f11 100644 --- a/src/main/java/org/codehaus/mojo/license/AbstractDownloadLicensesMojo.java +++ b/src/main/java/org/codehaus/mojo/license/AbstractDownloadLicensesMojo.java @@ -545,8 +545,12 @@ public void execute() getLog().warn( "There were " + downloadErrorCount + " download errors - check the warnings above" ); break; case xmlOutput: - throw new MojoFailureException( "There were " + downloadErrorCount + " download errors - check " - + licensesErrorsFile.getAbsolutePath() ); + if ( downloadErrorCount > 0 ) + { + throw new MojoFailureException( "There were " + downloadErrorCount + " download errors - check " + + licensesErrorsFile.getAbsolutePath() ); + } + break; default: throw new IllegalStateException( "Unexpected value of " + ErrorRemedy.class.getName() + ": " + errorRemedy ); From 798b87bcdb91f7cd1b65e0d477fcc3a0557b6511 Mon Sep 17 00:00:00 2001 From: Peter Palaga Date: Tue, 5 Feb 2019 22:16:56 +0100 Subject: [PATCH 2/2] Fix #240 Introduce url -> file name mapping --- src/it/ISSUE-154/expected_licenses.xml | 2 +- src/it/ISSUE-154/postbuild.groovy | 2 +- src/it/MLICENSE-24/invoker.properties | 2 +- .../invoker.properties | 2 + .../pom.xml | 67 +++++ .../postbuild.groovy | 30 ++ .../src/license/licenses.xml | 14 + .../java/org/codehaus/mojo/HelloWorld.java | 15 + .../invoker.properties | 1 + .../licenses-config.xml | 18 ++ .../licenses.expected.xml | 58 ++++ src/it/download-licenses-file-names/pom.xml | 103 +++++++ .../postbuild.groovy | 46 +++ .../prebuild.groovy | 31 ++ .../src/license/licenses.xml | 14 + .../java/org/codehaus/mojo/HelloWorld.java | 15 + .../generated-resources/licenses.xml | 18 ++ .../licenses.expected.xml | 2 +- .../download-licenses-force/postbuild.groovy | 2 +- .../invoker.properties | 2 +- .../license/AbstractDownloadLicensesMojo.java | 284 +++++++++++++----- .../codehaus/mojo/license/download/Cache.java | 105 +++++++ .../mojo/license/download/FileNameEntry.java | 64 ++++ .../license/download/PreferredFileNames.java | 152 ++++++++++ .../codehaus/mojo/license/utils/FileUtil.java | 18 ++ .../mojo/license/utils/LicenseDownloader.java | 115 ++++--- 26 files changed, 1071 insertions(+), 111 deletions(-) create mode 100644 src/it/download-licenses-file-names-bad-sha1-force/invoker.properties create mode 100644 src/it/download-licenses-file-names-bad-sha1-force/pom.xml create mode 100644 src/it/download-licenses-file-names-bad-sha1-force/postbuild.groovy create mode 100644 src/it/download-licenses-file-names-bad-sha1-force/src/license/licenses.xml create mode 100644 src/it/download-licenses-file-names-bad-sha1-force/src/main/java/org/codehaus/mojo/HelloWorld.java create mode 100644 src/it/download-licenses-file-names/invoker.properties create mode 100644 src/it/download-licenses-file-names/licenses-config.xml create mode 100644 src/it/download-licenses-file-names/licenses.expected.xml create mode 100644 src/it/download-licenses-file-names/pom.xml create mode 100644 src/it/download-licenses-file-names/postbuild.groovy create mode 100644 src/it/download-licenses-file-names/prebuild.groovy create mode 100644 src/it/download-licenses-file-names/src/license/licenses.xml create mode 100644 src/it/download-licenses-file-names/src/main/java/org/codehaus/mojo/HelloWorld.java create mode 100644 src/it/download-licenses-file-names/target-initial/generated-resources/licenses.xml create mode 100644 src/main/java/org/codehaus/mojo/license/download/Cache.java create mode 100644 src/main/java/org/codehaus/mojo/license/download/FileNameEntry.java create mode 100644 src/main/java/org/codehaus/mojo/license/download/PreferredFileNames.java diff --git a/src/it/ISSUE-154/expected_licenses.xml b/src/it/ISSUE-154/expected_licenses.xml index bc358c357..acfc33910 100644 --- a/src/it/ISSUE-154/expected_licenses.xml +++ b/src/it/ISSUE-154/expected_licenses.xml @@ -10,7 +10,7 @@ Public Domain http://repository.jboss.org/licenses/cc0-1.0.txt repo - public domain - cc0-1.0.txt + public domain - alternative.txt diff --git a/src/it/ISSUE-154/postbuild.groovy b/src/it/ISSUE-154/postbuild.groovy index e72c50547..ec843b266 100644 --- a/src/it/ISSUE-154/postbuild.groovy +++ b/src/it/ISSUE-154/postbuild.groovy @@ -24,6 +24,6 @@ file = new File(basedir, 'target/licenses.xml'); expectedFile = new File(basedir, 'expected_licenses.xml'); assert expectedFile.text.equals(file.text); -licenseFile = new File(basedir, 'target/generated-resources/licenses/public domain - cc0-1.0.txt'); +licenseFile = new File(basedir, 'target/generated-resources/licenses/public domain - alternative.txt'); expectedLicenseTxt = new File(basedir, 'src/alternative.txt'); assert expectedLicenseTxt.text.equals(licenseFile.text); diff --git a/src/it/MLICENSE-24/invoker.properties b/src/it/MLICENSE-24/invoker.properties index bce8438cb..32494184d 100644 --- a/src/it/MLICENSE-24/invoker.properties +++ b/src/it/MLICENSE-24/invoker.properties @@ -1 +1 @@ -invoker.goals=clean validate \ No newline at end of file +invoker.goals=clean validate -X -e \ No newline at end of file diff --git a/src/it/download-licenses-file-names-bad-sha1-force/invoker.properties b/src/it/download-licenses-file-names-bad-sha1-force/invoker.properties new file mode 100644 index 000000000..5efdb5bce --- /dev/null +++ b/src/it/download-licenses-file-names-bad-sha1-force/invoker.properties @@ -0,0 +1,2 @@ +invoker.goals = clean license:download-licenses -Dlicense.forceDownload=true -Dlicense.errorRemedy=xmlOutput -Dlicense.sortByGroupIdAndArtifactId=true -e -X +invoker.buildResult=failure diff --git a/src/it/download-licenses-file-names-bad-sha1-force/pom.xml b/src/it/download-licenses-file-names-bad-sha1-force/pom.xml new file mode 100644 index 000000000..ecf5dac7e --- /dev/null +++ b/src/it/download-licenses-file-names-bad-sha1-force/pom.xml @@ -0,0 +1,67 @@ + + + 4.0.0 + + org.codehaus.mojo.license + download-licenses-file-names-bad-sha1 + 1.0-SNAPSHOT + Integration Test + http://maven.apache.org + + Check -Dlicense.errorRemedy=xmlOutput -Dlicense.sortByGroupIdAndArtifactId=true + + + + + antlr + antlr + 2.7.7 + + + * + * + + + + + commons-collections + commons-collections + 3.2.2 + + + * + * + + + + + + + + + org.codehaus.mojo + license-maven-plugin + @pom.version@ + + + + download-licenses + + + + + + + sha1:2b8b815229aa8a61e483fb4ba0588b8b6c491890 + \Qhttp://www.apache.org/licenses/LICENSE-2.0.txt\E + \Qhttps://www.apache.org/licenses/LICENSE-2.0.txt\E + \Qhttp://www.antlr.org/license.html\E + + + + + + + diff --git a/src/it/download-licenses-file-names-bad-sha1-force/postbuild.groovy b/src/it/download-licenses-file-names-bad-sha1-force/postbuild.groovy new file mode 100644 index 000000000..d92f2289a --- /dev/null +++ b/src/it/download-licenses-file-names-bad-sha1-force/postbuild.groovy @@ -0,0 +1,30 @@ +/* + * #%L + * License Maven Plugin + * %% + * Copyright (C) 2008 - 2011 CodeLutin, Codehaus, Tony Chemit, Tony chemit + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ + +import java.nio.file.Path; +import java.nio.file.Files; + +final Path basePath = basedir.toPath() + +final Path log = basePath.resolve('build.log') +assert Files.exists(log) +assert log.text.contains('URL \'http://www.antlr.org/license.html\' returned content with unexpected sha1 \'074f8d6e91730b40875178666513014472888247\'; expected \'2b8b815229aa8a61e483fb4ba0588b8b6c491890\'') diff --git a/src/it/download-licenses-file-names-bad-sha1-force/src/license/licenses.xml b/src/it/download-licenses-file-names-bad-sha1-force/src/license/licenses.xml new file mode 100644 index 000000000..0c3d63fa2 --- /dev/null +++ b/src/it/download-licenses-file-names-bad-sha1-force/src/license/licenses.xml @@ -0,0 +1,14 @@ + + + + commons-logging + commons-logging + + + Apache License 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + \ No newline at end of file diff --git a/src/it/download-licenses-file-names-bad-sha1-force/src/main/java/org/codehaus/mojo/HelloWorld.java b/src/it/download-licenses-file-names-bad-sha1-force/src/main/java/org/codehaus/mojo/HelloWorld.java new file mode 100644 index 000000000..16aaa7fed --- /dev/null +++ b/src/it/download-licenses-file-names-bad-sha1-force/src/main/java/org/codehaus/mojo/HelloWorld.java @@ -0,0 +1,15 @@ +package org.codehaus.mojo; + +public class HelloWorld +{ + + /** + * @param args + */ + public static void main( String[] args ) + { + // TODO Auto-generated method stub + + } + +} diff --git a/src/it/download-licenses-file-names/invoker.properties b/src/it/download-licenses-file-names/invoker.properties new file mode 100644 index 000000000..b0b7d7993 --- /dev/null +++ b/src/it/download-licenses-file-names/invoker.properties @@ -0,0 +1 @@ +invoker.goals = clean license:download-licenses -DlicensesConfigFile=./licenses-config.xml -Dlicense.errorRemedy=xmlOutput -Dlicense.sortByGroupIdAndArtifactId=true -e -X diff --git a/src/it/download-licenses-file-names/licenses-config.xml b/src/it/download-licenses-file-names/licenses-config.xml new file mode 100644 index 000000000..40db2f219 --- /dev/null +++ b/src/it/download-licenses-file-names/licenses-config.xml @@ -0,0 +1,18 @@ + + + + + antlr + antlr + 2.7.7 + + + BSD License + http://www.antlr.org/license.html + repo + bsd license - license.html + + + + + diff --git a/src/it/download-licenses-file-names/licenses.expected.xml b/src/it/download-licenses-file-names/licenses.expected.xml new file mode 100644 index 000000000..0839cb76d --- /dev/null +++ b/src/it/download-licenses-file-names/licenses.expected.xml @@ -0,0 +1,58 @@ + + + + + antlr + antlr + 2.7.7 + + + BSD License + http://www.antlr.org/license.html + repo + bsd-antlr.html + + + + + com.sun.activation + javax.activation + 1.2.0 + + + CDDL/GPLv2+CE + https://github.com/javaee/activation/blob/master/LICENSE.txt + repo + cddl-gplv2-ce.txt + CDDL or GPL version 2 plus the Classpath Exception + + + + + commons-beanutils + commons-beanutils + 1.9.3 + + + Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + asl2.txt + + + + + commons-collections + commons-collections + 3.2.2 + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + asl2.txt + + + + + diff --git a/src/it/download-licenses-file-names/pom.xml b/src/it/download-licenses-file-names/pom.xml new file mode 100644 index 000000000..706d22a0e --- /dev/null +++ b/src/it/download-licenses-file-names/pom.xml @@ -0,0 +1,103 @@ + + + 4.0.0 + + org.codehaus.mojo.license + download-licenses-file-names + 1.0-SNAPSHOT + Integration Test + http://maven.apache.org + + Check -DlicensesConfigFile=./licenses-config.xml -Dlicense.errorRemedy=xmlOutput -Dlicense.sortByGroupIdAndArtifactId=true + + + + + antlr + antlr + 2.7.7 + + + * + * + + + + + commons-collections + commons-collections + 3.2.2 + + + * + * + + + + + com.sun.activation + javax.activation + 1.2.0 + + + * + * + + + + + commons-beanutils + commons-beanutils + 1.9.3 + + + * + * + + + + + + + + + + org.codehaus.mojo + license-maven-plugin + @pom.version@ + + + + download-licenses + + + + + + + ^https?://github\.com/([^/]+)/([^/]+)/blob/(.*)$ + https://raw.githubusercontent.com/$1/$2/$3 + + + + + sha1:074f8d6e91730b40875178666513014472888247 + \Qhttp://www.antlr.org/license.html\E + + + sha1:2b8b815229aa8a61e483fb4ba0588b8b6c491890 + \Qhttp://www.apache.org/licenses/LICENSE-2.0.txt\E + \Qhttps://www.apache.org/licenses/LICENSE-2.0.txt\E + + + sha1:c27730aef98e1d50e937d808654daad00b258f90 + \Qhttps://raw.githubusercontent.com/javaee/activation/master/LICENSE.txt\E + + + + + + + diff --git a/src/it/download-licenses-file-names/postbuild.groovy b/src/it/download-licenses-file-names/postbuild.groovy new file mode 100644 index 000000000..5d2bfd7c9 --- /dev/null +++ b/src/it/download-licenses-file-names/postbuild.groovy @@ -0,0 +1,46 @@ +/* + * #%L + * License Maven Plugin + * %% + * Copyright (C) 2008 - 2011 CodeLutin, Codehaus, Tony Chemit, Tony chemit + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ + +import java.nio.file.Path; +import java.nio.file.Files; + +final Path basePath = basedir.toPath() + +assert !Files.exists(basePath.resolve('target/generated-resources/licenses/apache license 2.0 - license-2.0.txt')) +assert !Files.exists(basePath.resolve('target/generated-resources/licenses/bsd license - license.html')) + +final Path asl2 = basePath.resolve('target/generated-resources/licenses/asl2.txt') +assert Files.exists(asl2) +assert asl2.text.contains('Version 2.0, January 2004') + +final Path bsdAntlr = basePath.resolve('target/generated-resources/licenses/bsd-antlr.html') +assert Files.exists(bsdAntlr) +assert bsdAntlr.text.contains('Copyright (c) 2012 Terence Parr and Sam Harwell') + +final Path cddl = basePath.resolve('target/generated-resources/licenses/cddl-gplv2-ce.txt') +assert Files.exists(cddl) +assert new String(Files.readAllBytes(cddl), 'ISO-8859-1').contains('COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.1') + +final Path expectedLicensesXml = basePath.resolve('licenses.expected.xml') +final Path licensesXml = basePath.resolve('target/generated-resources/licenses.xml') +assert expectedLicensesXml.text.equals(licensesXml.text) + diff --git a/src/it/download-licenses-file-names/prebuild.groovy b/src/it/download-licenses-file-names/prebuild.groovy new file mode 100644 index 000000000..e555998c9 --- /dev/null +++ b/src/it/download-licenses-file-names/prebuild.groovy @@ -0,0 +1,31 @@ +/* + * #%L + * License Maven Plugin + * %% + * Copyright (C) 2008 - 2011 CodeLutin, Codehaus, Tony Chemit, Tony chemit + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ + +import java.nio.file.Path; +import java.nio.file.Files; + +Path basePath = basedir.toPath() + +Files.move(basePath.resolve('target-initial'), basePath.resolve('target')) + +Path licenses = basePath.resolve('target/generated-resources/licenses.xml') +assert Files.exists(licenses) diff --git a/src/it/download-licenses-file-names/src/license/licenses.xml b/src/it/download-licenses-file-names/src/license/licenses.xml new file mode 100644 index 000000000..0c3d63fa2 --- /dev/null +++ b/src/it/download-licenses-file-names/src/license/licenses.xml @@ -0,0 +1,14 @@ + + + + commons-logging + commons-logging + + + Apache License 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + \ No newline at end of file diff --git a/src/it/download-licenses-file-names/src/main/java/org/codehaus/mojo/HelloWorld.java b/src/it/download-licenses-file-names/src/main/java/org/codehaus/mojo/HelloWorld.java new file mode 100644 index 000000000..16aaa7fed --- /dev/null +++ b/src/it/download-licenses-file-names/src/main/java/org/codehaus/mojo/HelloWorld.java @@ -0,0 +1,15 @@ +package org.codehaus.mojo; + +public class HelloWorld +{ + + /** + * @param args + */ + public static void main( String[] args ) + { + // TODO Auto-generated method stub + + } + +} diff --git a/src/it/download-licenses-file-names/target-initial/generated-resources/licenses.xml b/src/it/download-licenses-file-names/target-initial/generated-resources/licenses.xml new file mode 100644 index 000000000..e19c7bdd4 --- /dev/null +++ b/src/it/download-licenses-file-names/target-initial/generated-resources/licenses.xml @@ -0,0 +1,18 @@ + + + + + commons-collections + commons-collections + 3.2.2 + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + the apache software license, version 2.0 - license-2.0.txt + + + + + diff --git a/src/it/download-licenses-force/licenses.expected.xml b/src/it/download-licenses-force/licenses.expected.xml index e35844617..fe856313f 100644 --- a/src/it/download-licenses-force/licenses.expected.xml +++ b/src/it/download-licenses-force/licenses.expected.xml @@ -35,7 +35,7 @@ GNU Lesser General Public License v2.1 or later http://www.opensource.org/licenses/LGPL-2.1 - gnu lesser general public license v2.1 or later - lgpl-2.1.txt.html + gnu lesser general public license v2.1 or later - lgpl-2.1.html See also: http://hibernate.org/license diff --git a/src/it/download-licenses-force/postbuild.groovy b/src/it/download-licenses-force/postbuild.groovy index 5d0f7221d..db2917e85 100644 --- a/src/it/download-licenses-force/postbuild.groovy +++ b/src/it/download-licenses-force/postbuild.groovy @@ -30,7 +30,7 @@ assert Files.exists(asl2) assert !asl2.text.contains('This content is fake.') assert asl2.text.contains('Version 2.0, January 2004') -final Path lgpl21 = basePath.resolve('target/generated-resources/licenses/gnu lesser general public license v2.1 or later - lgpl-2.1.txt.html') +final Path lgpl21 = basePath.resolve('target/generated-resources/licenses/gnu lesser general public license v2.1 or later - lgpl-2.1.html') assert Files.exists(lgpl21) assert lgpl21.text.contains('Version 2.1, February 1999') diff --git a/src/it/download-licenses-include-exclude-types/invoker.properties b/src/it/download-licenses-include-exclude-types/invoker.properties index 2bdc012db..441ff83dd 100644 --- a/src/it/download-licenses-include-exclude-types/invoker.properties +++ b/src/it/download-licenses-include-exclude-types/invoker.properties @@ -1 +1 @@ -invoker.goals = clean validate +invoker.goals = clean validate -X diff --git a/src/main/java/org/codehaus/mojo/license/AbstractDownloadLicensesMojo.java b/src/main/java/org/codehaus/mojo/license/AbstractDownloadLicensesMojo.java index 0e2a39f11..6e48a1a36 100644 --- a/src/main/java/org/codehaus/mojo/license/AbstractDownloadLicensesMojo.java +++ b/src/main/java/org/codehaus/mojo/license/AbstractDownloadLicensesMojo.java @@ -34,6 +34,10 @@ import org.apache.maven.settings.Proxy; import org.codehaus.mojo.license.api.DependenciesTool; import org.codehaus.mojo.license.api.MavenProjectDependenciesConfigurator; +import org.codehaus.mojo.license.api.ResolvedProjectDependencies; +import org.codehaus.mojo.license.download.Cache; +import org.codehaus.mojo.license.download.FileNameEntry; +import org.codehaus.mojo.license.download.PreferredFileNames; import org.codehaus.mojo.license.model.ProjectLicense; import org.codehaus.mojo.license.model.ProjectLicenseInfo; import org.codehaus.mojo.license.utils.FileUtil; @@ -48,9 +52,10 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; import java.nio.charset.Charset; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; @@ -64,8 +69,6 @@ import java.util.SortedMap; import java.util.regex.Pattern; -import org.codehaus.mojo.license.api.ResolvedProjectDependencies; - /** * Created on 23/05/16. * @@ -354,6 +357,51 @@ public abstract class AbstractDownloadLicensesMojo @Parameter private List licenseUrlReplacements; + /** + * A map that helps to select names for the local files where the downloaded licenses are stored. + *

+ * Keys in the map are local file names. These files will be created under {@link #licensesOutputDirectory}. + *

+ * Values are white space ({@code " \t\n\r"}) separated lists of regular expressions that will be used to match + * license URLs. The regular expressions are compiled using {@link Pattern#CASE_INSENSITIVE}. Note that various + * characters that commonly occur in URLs have special meanings in regular extensions. Therefore, consider using + * regex quoting as described in {@link Pattern}. + *

+ * In addition to URL patterns, the list must also contain the sha1 checksum of the expected content. This is to + * ensure that URLs returning distinct licenses cannot be saved under the same local path. Note that strict checking + * of the checksum happens only when {@link #forceDownload} is {@code true}. Otherwise the mojo assumes the URL + * -> local name mapping is correct and downloads from the URL only if the local file does not exist. + *

+ * Using {@link #licenseUrlFileNames} for URLs returning volatile content is not a good idea. + *

An example: + * 

+     * {@code
+     * 
+     *   
+     *       sha1:81ffbd1712afe8cdf138b570c0fc9934742c33c1
+     *       https?://(www\.)?antlr\.org/license\.html
+     *   
+     *   
+     *       sha1:534a3fc9ae1076409bb00d01127dbba1e2620e92
+     *       \Qhttps://raw.githubusercontent.com/javaee/activation/master/LICENSE.txt\E
+     *   
+     * 
+     * }
+     *
+ *

+ * Relationship to other parameters: + *

    + *
  • {@link #licenseUrlReplacements} are applied before {@link #licenseUrlFileNames}
    • + *
  • {@link #licenseUrlFileNames} have higher precedence than {@code } elements in + * {@link #licensesConfigFile}
    • + *
  • {@link #licenseUrlFileNames} are ignored when {@link #organizeLicensesByDependencies} is {@code true}
    • + *
+ * + * @since 1.18 + */ + @Parameter + private Map licenseUrlFileNames; + // ---------------------------------------------------------------------- // Plexus Components // ---------------------------------------------------------------------- @@ -369,13 +417,13 @@ public abstract class AbstractDownloadLicensesMojo // ---------------------------------------------------------------------- // Private Fields // ---------------------------------------------------------------------- - + private PreferredFileNames preferredFileNames; /** * A map from the license URLs to file names (without path) where the * licenses were downloaded. This helps the plugin to avoid downloading * the same license multiple times. */ - private Map downloadedLicenseURLs = new HashMap<>(); + private Cache cache; /** * Proxy Login/Password encoded(only if usgin a proxy with authentication). @@ -436,6 +484,8 @@ public void execute() } this.errorRemedy = getEffectiveErrorRemedy( this.quiet, this.errorRemedy ); + this.preferredFileNames = PreferredFileNames.build( licensesOutputDirectory, licenseUrlFileNames, getLog() ); + this.cache = new Cache( licenseUrlFileNames != null && !licenseUrlFileNames.isEmpty() ); initDirectories(); @@ -479,14 +529,27 @@ public void execute() { depProject = createDependencyProject( project ); } - if ( !offline ) + depProjectLicenses.add( depProject ); + } + if ( !offline ) + { + /* First save the matching URLs into the cache */ + for ( ProjectLicenseInfo depProject : depProjectLicenses ) { - downloadLicenses( licenseDownloader, depProject ); + downloadLicenses( licenseDownloader, depProject, true ); + } + getLog().debug( "Finished populating cache" ); + /* + * Then attempt to download the rest of the URLs using the available cache entries to select local + * file names based on file content sha1 + */ + for ( ProjectLicenseInfo depProject : depProjectLicenses ) + { + downloadLicenses( licenseDownloader, depProject, false ); } - depProjectLicenses.add( depProject ); } } - catch ( Exception e ) + catch ( IOException e ) { throw new RuntimeException( e ); } @@ -793,14 +856,30 @@ private void loadLicenseInfo( Map configuredDepLicen { for ( ProjectLicense license : dep.getLicenses() ) { - final String fileName = license.getFile(); - if ( fileName != null ) + final String url = license.getUrl(); + if ( url != null ) { - final File licenseFile = new File( licensesOutputDirectory, fileName ); - if ( licenseFile.exists() ) + final String licenseUrl = rewriteLicenseUrlIfNecessary( url ); + final FileNameEntry fileNameEntry = + getLicenseFileName( dep, licenseUrl, license.getName(), license.getFile() ); + final File licenseFile = fileNameEntry.getFile(); + if ( !forceDownload && licenseFile.exists() ) { + final String actualSha1 = FileUtil.sha1( licenseFile.toPath() ); + if ( fileNameEntry.getSha1() != null && !actualSha1.equals( fileNameEntry.getSha1() ) ) + { + throw new MojoFailureException( "Unexpected sha1 checksum for file '" + + licenseFile.getAbsolutePath() + "': '" + actualSha1 + "'; expected '" + + fileNameEntry.getSha1() + "'. You may want to (a) re-run the current mojo" + + " with -Dlicense.forceDownload=true or (b) change the expected sha1 in" + + " the licenseUrlFileNames entry '" + + fileNameEntry.getFile().getName() + "' or (c) split the entry so that" + + " its URLs return content with different sha1 sums." ); + } // Save the URL so we don't download it again - downloadedLicenseURLs.put( license.getUrl(), licenseFile ); + cache.put( license.getUrl(), LicenseDownloadResult.success( licenseFile, + actualSha1, + fileNameEntry.isPreferred() ) ); } } } @@ -854,18 +933,25 @@ private ProjectLicenseInfo createDependencyProject( MavenProject depMavenProject * @param depProject the project containing the license * @param licenseUrl the license url * @param licenseName the license name + * @param string * @return A filename to be used for the downloaded license file + * @throws URISyntaxException */ - private String getLicenseFileName( ProjectLicenseInfo depProject, final URL licenseUrl, final String licenseName ) + private FileNameEntry getLicenseFileName( ProjectLicenseInfo depProject, final String url, + final String licenseName, String licenseFileName ) + throws URISyntaxException { - String defaultExtension = ".txt"; + final URI licenseUrl = new URI( url ); File licenseUrlFile = new File( licenseUrl.getPath() ); - String licenseFileName; - if ( organizeLicensesByDependencies ) { + if ( licenseFileName != null && !licenseFileName.isEmpty() ) + { + return new FileNameEntry( new File( licensesOutputDirectory, new File( licenseFileName ).getName() ), + false, null ); + } licenseFileName = String.format( "%s.%s%s", depProject.getGroupId(), depProject.getArtifactId(), licenseName != null ? "_" + licenseName @@ -873,6 +959,19 @@ private String getLicenseFileName( ProjectLicenseInfo depProject, final URL lice } else { + final FileNameEntry preferredFileNameEntry = preferredFileNames.getEntryByUrl( url ); + if ( preferredFileNameEntry != null ) + { + return preferredFileNameEntry; + } + + if ( licenseFileName != null && !licenseFileName.isEmpty() ) + { + return new FileNameEntry( new File( licensesOutputDirectory, + new File( licenseFileName ).getName() ), + false, null ); + } + licenseFileName = licenseUrlFile.getName(); if ( licenseName != null ) @@ -880,13 +979,6 @@ private String getLicenseFileName( ProjectLicenseInfo depProject, final URL lice licenseFileName = licenseName + " - " + licenseUrlFile.getName(); } - // Check if the file has a valid file extention - int extensionIndex = licenseFileName.lastIndexOf( "." ); - if ( extensionIndex == -1 || extensionIndex > ( licenseFileName.length() - 3 ) ) - { - // This means it isn't a valid file extension, so append the default - licenseFileName = licenseFileName + defaultExtension; - } // Normalize whitespace licenseFileName = licenseFileName.replaceAll( "\\s+", " " ); } @@ -894,23 +986,25 @@ private String getLicenseFileName( ProjectLicenseInfo depProject, final URL lice // lower case and (back)slash removal licenseFileName = licenseFileName.toLowerCase( Locale.US ).replaceAll( "[\\\\/]+", "_" ); - return licenseFileName; + return new FileNameEntry( new File( licensesOutputDirectory, licenseFileName ), false, null ); } /** * Download the licenses associated with this project * * @param depProject The project which generated the dependency + * @param matchingUrlsOnly * @throws MojoFailureException */ - private void downloadLicenses( LicenseDownloader licenseDownloader, ProjectLicenseInfo depProject ) - throws MojoFailureException + private void downloadLicenses( LicenseDownloader licenseDownloader, ProjectLicenseInfo depProject, + boolean matchingUrlsOnly ) + throws MojoFailureException { getLog().debug( "Downloading license(s) for project " + depProject ); List licenses = depProject.getLicenses(); - if ( depProject.getLicenses() == null || depProject.getLicenses().isEmpty() ) + if ( matchingUrlsOnly && ( depProject.getLicenses() == null || depProject.getLicenses().isEmpty() ) ) { handleError( depProject, "No license information available for: " + depProject ); return; @@ -919,75 +1013,109 @@ private void downloadLicenses( LicenseDownloader licenseDownloader, ProjectLicen int licenseIndex = 0; for ( ProjectLicense license : licenses ) { - if ( license.getUrl() == null ) + if ( matchingUrlsOnly && license.getUrl() == null ) { handleError( depProject, "No URL for license at index " + licenseIndex + " in dependency " - + depProject.toString() ); + + depProject.toString() ); } - else + else if ( license.getUrl() != null ) { final String licenseUrl = rewriteLicenseUrlIfNecessary( license.getUrl() ); + + final LicenseDownloadResult cachedResult = cache.get( licenseUrl ); try { - File licenseOutputFile = downloadedLicenseURLs.get( licenseUrl ); - if ( licenseOutputFile == null ) + if ( cachedResult != null ) { - final String licenseFileName; - if ( license.getFile() != null ) - { - licenseFileName = new File( license.getFile() ).getName(); - } - else + if ( cachedResult.isPreferredFileName() == matchingUrlsOnly ) { - licenseFileName = getLicenseFileName( depProject, - new URL( license.getUrl() ), - license.getName() ); + if ( organizeLicensesByDependencies ) + { + final FileNameEntry fileNameEntry = + getLicenseFileName( depProject, licenseUrl, license.getName(), license.getFile() ); + final File cachedFile = cachedResult.getFile(); + final LicenseDownloadResult byDepsResult; + final File byDepsFile = fileNameEntry.getFile(); + if ( cachedResult.isSuccess() && !cachedFile.equals( byDepsFile ) ) + { + Files.copy( cachedFile.toPath(), byDepsFile.toPath() ); + byDepsResult = cachedResult.withFile( byDepsFile ); + } + else + { + byDepsResult = cachedResult; + } + handleResult( licenseUrl, byDepsResult, depProject, license ); + } + else + { + handleResult( licenseUrl, cachedResult, depProject, license ); + } } - licenseOutputFile = new File( licensesOutputDirectory, licenseFileName ); + return; } - - if ( !licenseOutputFile.exists() || forceDownload ) + else { - if ( !downloadedLicenseURLs.containsKey( licenseUrl ) || organizeLicensesByDependencies ) - { - final LicenseDownloadResult result = - licenseDownloader.downloadLicense( licenseUrl, proxyLoginPasswordEncoded, - licenseOutputFile, getLog() ); + /* No cache entry for the current URL */ + final FileNameEntry fileNameEntry = + getLicenseFileName( depProject, licenseUrl, license.getName(), license.getFile() ); - if ( result.isSuccess() ) + final File licenseOutputFile = fileNameEntry.getFile(); + if ( matchingUrlsOnly == fileNameEntry.isPreferred() ) + { + if ( !licenseOutputFile.exists() || forceDownload ) { - licenseOutputFile = result.getFile(); - downloadedLicenseURLs.put( licenseUrl, licenseOutputFile ); + LicenseDownloadResult result = + licenseDownloader.downloadLicense( licenseUrl, proxyLoginPasswordEncoded, + fileNameEntry, getLog() ); + if ( !organizeLicensesByDependencies && result.isSuccess() ) + { + /* check if we can re-use an existing file that has the same content */ + final String name = preferredFileNames.getFileNameBySha1( result.getSha1() ); + if ( name != null ) + { + final File oldFile = result.getFile(); + if ( !oldFile.getName().equals( name ) ) + { + final File newFile = new File( licensesOutputDirectory, name ); + result = result.withFile( newFile ); + oldFile.delete(); + } + } + } + handleResult( licenseUrl, result, depProject, license ); + cache.put( licenseUrl, result ); } - else + else if ( licenseOutputFile.exists() ) { - handleError( depProject, result.getErrorMessage() ); + final LicenseDownloadResult result = + LicenseDownloadResult.success( licenseOutputFile, + FileUtil.sha1( licenseOutputFile.toPath() ), + fileNameEntry.isPreferred() ); + handleResult( licenseUrl, result, depProject, license ); + cache.put( licenseUrl, result ); } - } } - - if ( licenseOutputFile != null ) - { - license.setFile( licenseOutputFile.getName() ); - } - } catch ( URISyntaxException e ) { handleError( depProject, "POM for dependency " + depProject.toString() + " has an invalid license URL: " + licenseUrl ); + getLog().debug( e ); } catch ( FileNotFoundException e ) { handleError( depProject, "POM for dependency " + depProject.toString() - + " has a license URL that returns file not found: " + licenseUrl ); + + " has a license URL that returns file not found: " + licenseUrl ); + getLog().debug( e ); } catch ( IOException e ) { handleError( depProject, "Unable to retrieve license from URL '" + licenseUrl + "' for dependency '" + depProject.toString() + "': " + e.getMessage() ); + getLog().debug( e ); } } licenseIndex++; @@ -995,6 +1123,20 @@ private void downloadLicenses( LicenseDownloader licenseDownloader, ProjectLicen } + private void handleResult( String licenseUrl, LicenseDownloadResult result, ProjectLicenseInfo depProject, + ProjectLicense license ) + throws MojoFailureException + { + if ( result.isSuccess() ) + { + license.setFile( result.getFile().getName() ); + } + else + { + handleError( depProject, result.getErrorMessage() ); + } + } + private void handleError( ProjectLicenseInfo depProject, String msg ) throws MojoFailureException { switch ( errorRemedy ) @@ -1008,6 +1150,7 @@ private void handleError( ProjectLicenseInfo depProject, String msg ) throws Moj case failFast: throw new MojoFailureException( msg ); case xmlOutput: + getLog().debug( msg ); depProject.addDownloaderMessage( msg ); break; default: @@ -1045,18 +1188,21 @@ private String rewriteLicenseUrlIfNecessary( final String originalLicenseUrl ) * * @since 1.18 */ - public static enum ErrorRemedy + public enum ErrorRemedy { /** All errors are ignored */ ignore, /** All errors are output to the log as warnings */ warn, - /** The first encountered error is logged and a {@link MojoFailureException} is - * thrown */ + /** + * The first encountered error is logged and a {@link MojoFailureException} is thrown + */ failFast, - /** Error messages are added as {@code } to + /** + * Error messages are added as {@code } to * {@link AbstractDownloadLicensesMojo#licensesErrorsFile}; in case there are error messages, the build will - * fail after processing all dependencies. */ + * fail after processing all dependencies. + */ xmlOutput } } diff --git a/src/main/java/org/codehaus/mojo/license/download/Cache.java b/src/main/java/org/codehaus/mojo/license/download/Cache.java new file mode 100644 index 000000000..b97c57198 --- /dev/null +++ b/src/main/java/org/codehaus/mojo/license/download/Cache.java @@ -0,0 +1,105 @@ +package org.codehaus.mojo.license.download; + +/* + * #%L + * License Maven Plugin + * %% + * Copyright (C) 2018 Codehaus + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ + +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.codehaus.mojo.license.utils.LicenseDownloader.LicenseDownloadResult; + +/** + * A simple {@link HashMap} based in-memory cache for storing {@link LicenseDownloadResult}s. + * + * @author Peter Palaga + * @since 1.18 + */ +public class Cache +{ + private final Map urlToFile = new HashMap<>(); + + private final Map sha1ToFile = new HashMap<>(); + + private final boolean enforcingUniqueSha1s; + + public Cache( boolean enforcingUniqueSha1s ) + { + super(); + this.enforcingUniqueSha1s = enforcingUniqueSha1s; + } + + /** + * @param url the cache key to seek + * @return the {@link LicenseDownloadResult} bound to the given {@code url} or {@code null} if no entry is bound to + * the given {@code url} + */ + public LicenseDownloadResult get( String url ) + { + return urlToFile.get( url ); + } + + /** + * Binds the given {@code url} to the give {@link LicenseDownloadResult}. If both {@link #enforcingUniqueSha1s} and + * {@link LicenseDownloadResult#isSuccess()} are {@code true} and an entry with the given + * {@link LicenseDownloadResult#getSha1()} already exists in {@link #sha1ToFile}, asserts that both the added and + * the available entry point at the same file. + * + * @param url the URL the given {@code entry} comes from + * @param entry the result of downloading from {@code url} + */ + public void put( String url, LicenseDownloadResult entry ) + { + if ( entry.isSuccess() ) + { + final String sha1 = entry.getSha1(); + final File entryFile = entry.getFile(); + final LicenseDownloadResult existing = sha1ToFile.get( sha1 ); + if ( existing == null ) + { + sha1ToFile.put( sha1, entry ); + } + else if ( enforcingUniqueSha1s && !existing.getFile().equals( entryFile ) ) + { + final File existingFile = existing.getFile(); + final StringBuilder sb = new StringBuilder(); + for ( Entry en : urlToFile.entrySet() ) + { + if ( existingFile.equals( en.getValue().getFile() ) ) + { + if ( sb.length() > 0 ) + { + sb.append( ", " ); + } + sb.append( en.getKey() ); + } + } + throw new IllegalStateException( "URL '" + url + + "' should belong to licenseUrlFileName having key '" + existingFile.getName() + + "' together with URLs '" + sb.toString() + "'" ); + } + } + urlToFile.put( url, entry ); + } + +} \ No newline at end of file diff --git a/src/main/java/org/codehaus/mojo/license/download/FileNameEntry.java b/src/main/java/org/codehaus/mojo/license/download/FileNameEntry.java new file mode 100644 index 000000000..aa6278f4a --- /dev/null +++ b/src/main/java/org/codehaus/mojo/license/download/FileNameEntry.java @@ -0,0 +1,64 @@ +package org.codehaus.mojo.license.download; + +/* + * #%L + * License Maven Plugin + * %% + * Copyright (C) 2018 Codehaus + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ + +import java.io.File; + +/** + * A triple consiting of a {@link File}, its SHA-1 checksum and a boolean whether {@link #file}'s name comes from + * {@link PreferredFileNames} and is thus preferred. + * + * @author Peter Palaga + * @since 1.18 + */ +public class FileNameEntry +{ + public FileNameEntry( File file, boolean preferred, String sha1 ) + { + super(); + this.file = file; + this.preferred = preferred; + this.sha1 = sha1; + } + + private final File file; + + private final boolean preferred; + + private final String sha1; + + public File getFile() + { + return file; + } + + public boolean isPreferred() + { + return preferred; + } + + public String getSha1() + { + return sha1; + } +} \ No newline at end of file diff --git a/src/main/java/org/codehaus/mojo/license/download/PreferredFileNames.java b/src/main/java/org/codehaus/mojo/license/download/PreferredFileNames.java new file mode 100644 index 000000000..3fd534f05 --- /dev/null +++ b/src/main/java/org/codehaus/mojo/license/download/PreferredFileNames.java @@ -0,0 +1,152 @@ +package org.codehaus.mojo.license.download; + +/* + * #%L + * License Maven Plugin + * %% + * Copyright (C) 2018 Codehaus + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ + +import java.io.File; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Pattern; + +import org.apache.maven.plugin.logging.Log; + +/** + * An url -> {@link FileNameEntry} mapping. + * + * @author Peter Palaga + * @since 1.18 + */ +public class PreferredFileNames +{ + /** + * @param licensesOutputDirectory + * @param licenseUrlFileNames + * @param log + * @return a new {@link PreferredFileNames} built of AbstractDownloadLicensesMojo.licenseUrlFileNames + */ + public static PreferredFileNames build( File licensesOutputDirectory, Map licenseUrlFileNames, + Log log ) + { + final Map>> fileNameToUrlPatterns = new LinkedHashMap<>(); + final Map sha1TofileName = new LinkedHashMap(); + if ( licenseUrlFileNames != null ) + { + for ( Entry en : licenseUrlFileNames.entrySet() ) + { + final String fileName = en.getKey(); + if ( fileName != null && !fileName.isEmpty() ) + { + if ( en.getValue() != null ) + { + String[] rawPatters = en.getValue().split( "\\s+" ); + if ( rawPatters != null ) + { + final List patterns = new ArrayList<>(); + String sha1 = null; + for ( String rawPattern : rawPatters ) + { + if ( rawPattern.startsWith( "sha1:" ) ) + { + if ( sha1 != null ) + { + throw new IllegalStateException( "sha1 defined twice for licenseFileName '" + + fileName + "'" ); + } + sha1 = rawPattern.substring( 5 ); + } + else + { + patterns.add( Pattern.compile( rawPattern, Pattern.CASE_INSENSITIVE ) ); + } + } + if ( sha1 == null ) + { + throw new IllegalStateException( "sha1 undefined for licenseFileName '" + fileName + + "'. Add 'sha1:' to the list of patterns '" + en.getValue() + + "'" ); + } + fileNameToUrlPatterns.put( fileName, + new AbstractMap.SimpleImmutableEntry<>( + sha1, + Collections.unmodifiableList( patterns ) ) ); + sha1TofileName.put( sha1, fileName ); + } + } + } + } + } + return new PreferredFileNames( licensesOutputDirectory, fileNameToUrlPatterns, sha1TofileName, log ); + + } + + private final File licensesOutputDirectory; + private final Map>> fileNameToUrlPatterns; + private final Map sha1ToFileName; + private final Log log; + + public PreferredFileNames( File licensesOutputDirectory, + Map>> fileNameToUrlPatterns, + Map sha1ToFileName, Log log ) + { + super(); + this.licensesOutputDirectory = licensesOutputDirectory; + this.fileNameToUrlPatterns = fileNameToUrlPatterns; + this.sha1ToFileName = sha1ToFileName; + this.log = log; + } + + /** + * @param sha1 the checksum to search by + * @return a file name bound the given {@code sha1} checksum or {@code null} + */ + public String getFileNameBySha1( String sha1 ) + { + return sha1ToFileName.get( sha1 ); + } + + /** + * @param url the URL to query + * @return the preferred {@link FileNameEntry} for the given {@code url} or {@code null} + */ + public FileNameEntry getEntryByUrl( String url ) + { + for ( Entry>> fn : fileNameToUrlPatterns.entrySet() ) + { + for ( Pattern pat : fn.getValue().getValue() ) + { + if ( pat.matcher( url ).matches() ) + { + log.debug( "Using file name '" + fn.getKey() + "' for URL '" + url + "' that matched pattern '" + + pat.pattern() + "'" ); + return new FileNameEntry( new File( licensesOutputDirectory, fn.getKey() ), true, + fn.getValue().getKey() ); + } + } + } + return null; + } +} \ No newline at end of file diff --git a/src/main/java/org/codehaus/mojo/license/utils/FileUtil.java b/src/main/java/org/codehaus/mojo/license/utils/FileUtil.java index 25747af96..02e616490 100644 --- a/src/main/java/org/codehaus/mojo/license/utils/FileUtil.java +++ b/src/main/java/org/codehaus/mojo/license/utils/FileUtil.java @@ -22,6 +22,7 @@ * #L% */ +import org.apache.commons.codec.binary.Hex; import org.codehaus.plexus.util.FileUtils; import org.codehaus.plexus.util.IOUtil; @@ -38,6 +39,10 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.StringReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -290,4 +295,17 @@ public int compare( File o1, File o2 ) } ); return result; } + + public static String sha1( Path in ) throws IOException + { + try + { + final MessageDigest md = MessageDigest.getInstance( "SHA-1" ); + return Hex.encodeHexString( md.digest( Files.readAllBytes( in ) ) ); + } + catch ( NoSuchAlgorithmException e ) + { + throw new RuntimeException( e ); + } + } } diff --git a/src/main/java/org/codehaus/mojo/license/utils/LicenseDownloader.java b/src/main/java/org/codehaus/mojo/license/utils/LicenseDownloader.java index 971ccdb2a..6f2fdfdcc 100644 --- a/src/main/java/org/codehaus/mojo/license/utils/LicenseDownloader.java +++ b/src/main/java/org/codehaus/mojo/license/utils/LicenseDownloader.java @@ -26,12 +26,15 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import org.apache.commons.codec.binary.Hex; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; import org.apache.http.StatusLine; @@ -41,7 +44,9 @@ import org.apache.http.entity.ContentType; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.maven.plugin.MojoFailureException; import org.apache.maven.plugin.logging.Log; +import org.codehaus.mojo.license.download.FileNameEntry; /** * Utilities for downloading remote license files. @@ -82,20 +87,25 @@ public LicenseDownloader() * @return the path to the file where the downloaded license file was stored * @throws IOException * @throws URISyntaxException + * @throws MojoFailureException */ - public LicenseDownloadResult downloadLicense( String licenseUrlString, String loginPassword, File outputFile, - Log log ) - throws IOException, URISyntaxException + public LicenseDownloadResult downloadLicense( String licenseUrlString, String loginPassword, + FileNameEntry fileNameEntry, Log log ) + throws IOException, URISyntaxException, MojoFailureException { + final File outputFile = fileNameEntry.getFile(); if ( licenseUrlString == null || licenseUrlString.length() == 0 ) { - return LicenseDownloadResult.success( outputFile ); + throw new IllegalArgumentException( "Null URL for file " + outputFile.getPath() ); } + log.debug( "Downloading " + licenseUrlString ); + if ( licenseUrlString.startsWith( "file://" ) ) { - Files.copy( Paths.get( new URI( licenseUrlString ) ), outputFile.toPath() ); - return LicenseDownloadResult.success( outputFile ); + Path in = Paths.get( new URI( licenseUrlString ) ); + Files.copy( in, outputFile.toPath() ); + return LicenseDownloadResult.success( outputFile, FileUtil.sha1( in ), fileNameEntry.isPreferred() ); } else { @@ -113,16 +123,40 @@ public LicenseDownloadResult downloadLicense( String licenseUrlString, String lo if ( entity != null ) { final ContentType contentType = ContentType.get( entity ); - File updatedFile = - updateFileExtension( outputFile, contentType != null ? contentType.getMimeType() : null ); + + File updatedFile = fileNameEntry.isPreferred() ? outputFile + : updateFileExtension( outputFile, + contentType != null ? contentType.getMimeType() : null ); try ( InputStream in = entity.getContent(); FileOutputStream fos = new FileOutputStream( updatedFile ) ) { - copyStream( in, fos ); + final MessageDigest md = MessageDigest.getInstance( "SHA-1" ); + final byte[] buf = new byte[1024]; + int len; + while ( ( len = in.read( buf ) ) >= 0 ) + { + md.update( buf, 0, len ); + fos.write( buf, 0, len ); + } + final String actualSha1 = Hex.encodeHexString( md.digest() ); + final String expectedSha1 = fileNameEntry.getSha1(); + if ( expectedSha1 != null && !expectedSha1.equals( actualSha1 ) ) + { + throw new MojoFailureException( "URL '" + licenseUrlString + + "' returned content with unexpected sha1 '" + actualSha1 + "'; expected '" + + expectedSha1 + "'. You may want to (a) re-run the current mojo" + + " with -Dlicense.forceDownload=true or (b) change the expected sha1 in" + + " the licenseUrlFileNames entry '" + + fileNameEntry.getFile().getName() + "' or (c) split the entry so that" + + " its URLs return content with different sha1 sums." ); + } + return LicenseDownloadResult.success( updatedFile, actualSha1, fileNameEntry.isPreferred() ); + } + catch ( NoSuchAlgorithmException e ) + { + throw new RuntimeException( e ); } - return LicenseDownloadResult.success( updatedFile ); - } else { @@ -132,24 +166,6 @@ public LicenseDownloadResult downloadLicense( String licenseUrlString, String lo } } - /** - * Copy data from one stream to another. - * - * @param inStream - * @param outStream - * @throws IOException - */ - private static void copyStream( InputStream inStream, OutputStream outStream ) - throws IOException - { - byte[] buf = new byte[1024]; - int len; - while ( ( len = inStream.read( buf ) ) > 0 ) - { - outStream.write( buf, 0, len ); - } - } - private static File updateFileExtension( File outputFile, String mimeType ) { final String realExtension = getFileExtension( mimeType ); @@ -161,6 +177,13 @@ private static File updateFileExtension( File outputFile, String mimeType ) return new File( outputFile.getAbsolutePath() + realExtension ); } } + /* default extension is .txt */ + final String name = outputFile.getName(); + final int periodPos = name.lastIndexOf( '.' ); + if ( periodPos < 0 || name.length() - periodPos > 5 ) + { + return new File( outputFile.getParent(), name.substring( 0, periodPos ) + ".txt" ); + } return outputFile; } @@ -191,8 +214,7 @@ private static String getFileExtension( String mimeType ) } @Override - public void close() - throws Exception + public void close() throws IOException { client.close(); } @@ -204,27 +226,33 @@ public void close() */ public static class LicenseDownloadResult { - public static LicenseDownloadResult success( File file ) + public static LicenseDownloadResult success( File file, String sha1, boolean preferredFileName ) { - return new LicenseDownloadResult( file, null ); + return new LicenseDownloadResult( file, sha1, preferredFileName, null ); } public static LicenseDownloadResult failure( String errorMessage ) { - return new LicenseDownloadResult( null, errorMessage ); + return new LicenseDownloadResult( null, null, false, errorMessage ); } - private LicenseDownloadResult( File file, String errorMessage ) + private LicenseDownloadResult( File file, String sha1, boolean preferredFileName, String errorMessage ) { super(); this.file = file; this.errorMessage = errorMessage; + this.sha1 = sha1; + this.preferredFileName = preferredFileName; } private final File file; private final String errorMessage; + private final String sha1; + + private final boolean preferredFileName; + public File getFile() { return file; @@ -239,6 +267,21 @@ public boolean isSuccess() { return errorMessage == null; } + + public boolean isPreferredFileName() + { + return preferredFileName; + } + + public String getSha1() + { + return sha1; + } + + public LicenseDownloadResult withFile( File otherFile ) + { + return new LicenseDownloadResult( otherFile, sha1, preferredFileName, errorMessage ); + } } }