RAT-377: Create a STANDARD process filter like the ARCHIVE process switch in RAT-372

apache-rat-core/src/main/java/org/apache/rat/Defaults.java

@@ -68,6 +68,8 @@ public final class Defaults {
 
     public static final ReportConfiguration.Processing ARCHIVE_PROCESSING = ReportConfiguration.Processing.NOTIFICATION;
 
+    public static final ReportConfiguration.Processing STANDARD_PROCESSING = ReportConfiguration.Processing.ABSENCE;
+
     public static final LicenseFilter LIST_FAMILIES = LicenseFilter.NONE;
 
     public static final LicenseFilter LIST_LICENSES = LicenseFilter.NONE;

apache-rat-core/src/main/java/org/apache/rat/Report.java

@@ -252,11 +252,20 @@ public String desc() {
      * @since 0.17.0
      */
     static final Option ARCHIVE = Option.builder().longOpt("archive").hasArg().argName("ProcessingType")
-            .desc(format("Specifies the level of detail in ARCHIVE reporting. (default is %s)",
+            .desc(format("Specifies the level of detail in ARCHIVE file reporting. (default is %s)",
                     ReportConfiguration.Processing.NOTIFICATION))
             .converter(s -> ReportConfiguration.Processing.valueOf(s.toUpperCase()))
             .build();
 
+    /**
+     * Specify the processing of STANDARD files.
+     */
+    static final Option STANDARD = Option.builder().longOpt("standard").hasArg().argName("ProcessingType")
+            .desc(format("Specifies the level of detail in STANDARD file reporting. (default is %s)",
+                    Defaults.STANDARD_PROCESSING))
+            .converter(s -> ReportConfiguration.Processing.valueOf(s.toUpperCase()))
+            .build();
+
     private static String asString(Object[] args) {
         return Arrays.stream(args).map(Object::toString).collect(Collectors.joining(", "));
     }
@@ -367,6 +376,14 @@ static ReportConfiguration createConfiguration(String baseDirectory, CommandLine
             }
         }
 
+        if (cl.hasOption(STANDARD)) {
+            try {
+                configuration.setStandardProcessing(cl.getParsedOptionValue(STANDARD));
+            } catch (ParseException e) {
+                logParseException(e, STANDARD, cl, Defaults.STANDARD_PROCESSING);
+            }
+        }
+
         if (cl.hasOption(OUT)) {
             try {
                 configuration.setOut((File) cl.getParsedOptionValue(OUT));
@@ -411,7 +428,6 @@ static ReportConfiguration createConfiguration(String baseDirectory, CommandLine
                 }
 
                 URL url = Report.class.getClassLoader().getResource(String.format("org/apache/rat/%s.xsl", style[0]));
-
                 IOSupplier<InputStream> ioSupplier = (url == null) ?
                         () -> Files.newInputStream(Paths.get(style[0])) :
                         url::openStream;
@@ -470,6 +486,7 @@ static FilenameFilter parseExclusions(List<String> excludes) {
     static Options buildOptions() {
         return new Options()
                 .addOption(ARCHIVE)
+                .addOption(STANDARD)
                 .addOption(DRY_RUN)
                 .addOption(LIST_FAMILIES)
                 .addOption(LIST_LICENSES)
@@ -514,7 +531,7 @@ static void printUsage(PrintWriter writer, Options opts) {
 
         String argumentPadding = createPadding(helpFormatter.getLeftPadding() + 5);
         for (Map.Entry<String, Supplier<String>> argInfo : ARGUMENT_TYPES.entrySet()) {
-            writer.format("\n<%s>\n", argInfo.getKey());
+            writer.format("%n<%s>%n", argInfo.getKey());
             helpFormatter.printWrapped(writer, helpFormatter.getWidth(), helpFormatter.getLeftPadding() + 10,
                     argumentPadding + argInfo.getValue().get());
         }

apache-rat-core/src/main/java/org/apache/rat/ReportConfiguration.java

@@ -58,7 +58,13 @@
  */
 public class ReportConfiguration {
 
-    public enum Processing {NOTIFICATION("List file as present"), PRESENCE("List any licenses found"), ABSENCE("List licenses found and any unknown licences");
+    public enum Processing {
+        /** List file as present only */
+        NOTIFICATION("List file as present"),
+        /** List all present licenses */
+        PRESENCE("List any licenses found"),
+        /** List all present licenses and unknown licenses */
+        ABSENCE("List licenses found and any unknown licences");
 
     private final String description;
     Processing(String description) {
@@ -86,7 +92,7 @@ public enum Processing {NOTIFICATION("List file as present"), PRESENCE("List any
     private LicenseFilter listLicenses;
     private boolean dryRun;
     private Processing archiveProcessing;
-
+    private Processing standardProcessing;
     /**
      * Constructor
      * @param log The Log implementation that messages will be written to.
@@ -121,6 +127,22 @@ public void setArchiveProcessing(Processing archiveProcessing) {
         this.archiveProcessing = archiveProcessing;
     }
 
+    /**
+     * Retrieves the archive processing type.
+     * @return The archive processing type.
+     */
+    public Processing getStandardProcessing() {
+        return standardProcessing == null ? Defaults.STANDARD_PROCESSING : standardProcessing;
+    }
+
+    /**
+     * Sets the archive processing type.  If not set will default to NOTIFICATION.
+     * @param standardProcessing the type of processing archives should have.
+     */
+    public void setStandardProcessing(Processing standardProcessing) {
+        this.standardProcessing = standardProcessing;
+    }
+
     /**
      * Retrieves the Log that was provided in the constructor.
      * @return the Log for the system.

apache-rat-core/src/main/java/org/apache/rat/analysis/DefaultAnalyserFactory.java

@@ -23,13 +23,17 @@
 import java.util.function.Predicate;
 
 import org.apache.rat.ConfigurationException;
+import org.apache.rat.Defaults;
+import org.apache.rat.Report;
 import org.apache.rat.ReportConfiguration;
 import org.apache.rat.api.Document;
 import org.apache.rat.api.RatException;
 import org.apache.rat.document.IDocumentAnalyser;
 import org.apache.rat.document.RatDocumentAnalysisException;
 import org.apache.rat.license.ILicense;
 import org.apache.rat.license.LicenseSetFactory;
+import org.apache.rat.report.ConfigurationReport;
+import org.apache.rat.utils.Log;
 import org.apache.rat.walker.ArchiveWalker;
 
 /**
@@ -74,25 +78,46 @@ public DefaultAnalyser(ReportConfiguration config, final Collection<ILicense> li
             this.configuration = config;
         }
 
+        /**
+         * Generates a predicate to filter out licenses that should not be reported.
+         * @param proc the processing status to filter.
+         * @return a Predicate to do the filtering.
+         */
+        private  Predicate<ILicense> licenseFilter(ReportConfiguration.Processing proc)  {
+            return license -> {
+                switch (proc) {
+                    case PRESENCE:
+                        return !license.getLicenseFamily().equals(UnknownLicense.INSTANCE.getLicenseFamily());
+                    case ABSENCE:
+                        return true;
+                    default:
+                        return false;
+                }
+            };
+        }
+
         @Override
         public void analyse(Document document) throws RatDocumentAnalysisException {
 
             TikaProcessor.process(configuration.getLog(), document);
+            Predicate<ILicense> licensePredicate = null;
 
             switch (document.getMetaData().getDocumentType()) {
             case STANDARD:
+                licensePredicate = licenseFilter(configuration.getStandardProcessing()).negate();
                 new DocumentHeaderAnalyser(configuration.getLog(), licenses).analyse(document);
+                if (configuration.getStandardProcessing() != Defaults.STANDARD_PROCESSING) {
+                    document.getMetaData().removeLicenses(licensePredicate);
+                }
                 break;
             case ARCHIVE:
+                licensePredicate = licenseFilter(configuration.getArchiveProcessing());
                 if (configuration.getArchiveProcessing() != ReportConfiguration.Processing.NOTIFICATION) {
                     ArchiveWalker archiveWalker = new ArchiveWalker(configuration, document);
-                    Predicate<ILicense> filter = configuration.getArchiveProcessing() == ReportConfiguration.Processing.ABSENCE ?
-                            l -> Boolean.TRUE : lic -> !lic.getLicenseFamily().equals(UnknownLicense.INSTANCE.getLicenseFamily());
                     try {
-                        Collection<Document> docs = archiveWalker.getDocuments(configuration.getLog());
-                        for (Document doc : docs) {
+                        for (Document doc : archiveWalker.getDocuments(configuration.getLog())) {
                             analyse(doc);
-                            doc.getMetaData().licenses().filter(filter).forEach(lic -> document.getMetaData().reportOnLicense(lic));
+                            doc.getMetaData().licenses().filter(licensePredicate).forEach(lic -> document.getMetaData().reportOnLicense(lic));
                         }
                     } catch (RatException e) {
                         throw new RatDocumentAnalysisException(e);

apache-rat-core/src/main/java/org/apache/rat/analysis/UnknownLicense.java

@@ -22,14 +22,16 @@
 import org.apache.rat.license.ILicenseFamily;
 import org.apache.rat.license.ILicenseFamilyBuilder;
 
+import java.util.Objects;
+
 /**
  * An ILicense implementation that represents an unknown license.
  * <p>
  * The UnknownLicense is used during processing to report that a document
  * license can not be determined.
  * </p>
  */
-public class UnknownLicense implements ILicense {
+public final class UnknownLicense implements ILicense {
 
     /**
      * The single instance of this class.
@@ -57,13 +59,18 @@ public void reset() {
     }
 
     @Override
-    public boolean matches(IHeaders headers) {
+    public boolean matches(final IHeaders headers) {
         return false;
     }
 
     @Override
-    public int compareTo(ILicense arg0) {
-        return getLicenseFamily().compareTo(arg0.getLicenseFamily());
+    public boolean equals(Object o) {
+        return super.equals(o);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(ILicenseFamily.UNKNOWN_CATEGORY);
     }
 
     @Override

apache-rat-core/src/main/java/org/apache/rat/analysis/matchers/SPDXMatcherFactory.java

@@ -19,8 +19,10 @@
 package org.apache.rat.analysis.matchers;
 
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -59,17 +61,20 @@ public class SPDXMatcherFactory {
      */
     private static Pattern groupSelector = Pattern.compile(".*SPDX-License-Identifier:\\s([A-Za-z0-9\\.\\-]+)");
 
-    /**
-     * the last line that this factory scanned.
-     */
-    private String lastLine;
     /**
      * The last matcer to match the line.
      */
-    private Match lastMatch;
+    private Set<String> lastMatch;
+
+    private boolean checked;
 
     private SPDXMatcherFactory() {
-        lastLine = null;
+        lastMatch = new HashSet<>();
+    }
+
+    private void reset() {
+        lastMatch.clear();
+        checked = false;
     }
 
     /**
@@ -103,16 +108,17 @@ private boolean check(String line, Match caller) {
         // if so then see if that name has been registered. If so then we have a match
         // and set
         // lastMatch.
-        if ((lastLine == null || !lastLine.equals(line)) && line.contains("SPDX-License-Identifier")) {
-            Matcher matcher = groupSelector.matcher(line);
-            if (matcher.find()) {
-                lastMatch = matchers.get(matcher.group(1));
-            } else {
-                lastMatch = null;
+        if (!checked) {
+            checked = true;
+            if (line.contains("SPDX-License-Identifier")) {
+                Matcher matcher = groupSelector.matcher(line);
+                while (matcher.find()) {
+                    lastMatch.add(matcher.group(1));
+                }
             }
         }
         // see if the caller matches lastMatch.
-        return (lastMatch != null) && caller.spdxId.equals(lastMatch.spdxId);
+        return lastMatch.contains(caller.spdxId);
     }
 
     @ConfigComponent(type = ComponentType.MATCHER, name = "spdx", desc = "Matches SPDX enclosed license identifier.")
@@ -144,7 +150,7 @@ public boolean matches(IHeaders headers) {
         @Override
         public void reset() {
             super.reset();
-            SPDXMatcherFactory.this.lastMatch = null;
+            SPDXMatcherFactory.this.reset();
         }
     }
 }

apache-rat-core/src/main/java/org/apache/rat/api/MetaData.java

@@ -22,6 +22,8 @@
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.apache.rat.license.ILicense;
@@ -155,6 +157,10 @@ public Document.Type getDocumentType() {
     public void reportOnLicense(ILicense license) {
         this.matchedLicenses.add(license);
     }
+
+    public void removeLicenses(Predicate<ILicense> filter) {
+        this.matchedLicenses.removeIf(filter);
+    }
 
     @Override
     public String toString() {

apache-rat-core/src/main/java/org/apache/rat/configuration/MatcherBuilderTracker.java

@@ -26,6 +26,7 @@
 import java.util.Objects;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.WordUtils;
 import org.apache.rat.ConfigurationException;
 import org.apache.rat.Defaults;
 import org.apache.rat.configuration.builders.AbstractBuilder;
@@ -34,7 +35,7 @@
  * A class to track the Matcher Builders as they are defined.  Matchers may be defined in multiple configuration files
  * this method tracks them so that they can be referenced across the configuration files.
  */
-public class MatcherBuilderTracker {
+public final class MatcherBuilderTracker {
 
     /** The instance of the BuildTracker. */
     public static MatcherBuilderTracker INSTANCE;
@@ -55,7 +56,7 @@ private static synchronized MatcherBuilderTracker instance() {
      * @param className the Class name for the builder.
      * @param name the short name for the builder. 
      */
-    public static void addBuilder(String className, String name) {
+    public static void addBuilder(final String className, final String name) {
         instance().addBuilderImpl(className, name);
     }
 
@@ -64,7 +65,7 @@ public static void addBuilder(String className, String name) {
      * @param name The name of the builder.
      * @return the builder for that name.
      */
-    public static AbstractBuilder getMatcherBuilder(String name) {
+    public static AbstractBuilder getMatcherBuilder(final String name) {
         Class<? extends AbstractBuilder> clazz = instance().matcherBuilders.get(name);
         if (clazz == null) {
             StringBuilder sb = new StringBuilder(System.lineSeparator()).append("Valid builders").append(System.lineSeparator());
@@ -94,7 +95,7 @@ public Collection<Class<? extends AbstractBuilder>> getClasses() {
         return Collections.unmodifiableCollection(matcherBuilders.values());
     }
 
-    private void addBuilderImpl(String className, String name) {
+    private void addBuilderImpl(final String className, String name) {
         Objects.requireNonNull(className, "className may not be null");
         Class<?> clazz;
         try {
@@ -116,7 +117,7 @@ private void addBuilderImpl(String className, String name) {
                     throw new ConfigurationException("Last segment of " + candidate.getName()
                             + " may not be 'Builder', but must end in 'Builder'");
                 }
-                name = name.replaceFirst(".", StringUtils.lowerCase(name.substring(0, 1)));
+                name = WordUtils.uncapitalize(name);
             }
             matcherBuilders.put(name, candidate);
         } else {

apache-rat-core/src/main/java/org/apache/rat/configuration/builders/ChildContainerBuilder.java

@@ -59,8 +59,8 @@ protected ChildContainerBuilder() {
      */
     public AbstractBuilder setResource(String resourceName) {
         URL url = this.getClass().getResource(resourceName);
-        try (final InputStream in = url.openStream()) {
-            BufferedReader buffer = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
+        try (final InputStream in = url.openStream();
+             BufferedReader buffer = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));) {
             String txt;
             while (null != (txt = buffer.readLine())) {
                 txt = txt.trim();