Merge branch 'master' into pr/3958

.all-contributorsrc

@@ -6639,6 +6639,51 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "lukelukes",
+      "name": "lukelukes",
+      "avatar_url": "https://avatars.githubusercontent.com/u/45536418?v=4",
+      "profile": "https://github.com/lukelukes",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "vibhory2j",
+      "name": "Vibhor Goyal",
+      "avatar_url": "https://avatars.githubusercontent.com/u/15845016?v=4",
+      "profile": "https://github.com/vibhory2j",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "Ramel0921",
+      "name": "Ramel0921",
+      "avatar_url": "https://avatars.githubusercontent.com/u/104978096?v=4",
+      "profile": "https://github.com/Ramel0921",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "flyhard",
+      "name": "Per Abich",
+      "avatar_url": "https://avatars.githubusercontent.com/u/409466?v=4",
+      "profile": "https://github.com/flyhard",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "filipponova",
+      "name": "Filippo Nova",
+      "avatar_url": "https://avatars.githubusercontent.com/u/12506636?v=4",
+      "profile": "https://github.com/filipponova",
+      "contributions": [
+        "bug"
+      ]
     }
   ],
   "contributorsPerLine": 7,

docs/pages/pmd/languages/html.md

@@ -14,5 +14,11 @@ last_updated: April 2022 (6.45.0)
 
 The HTML language module uses [jsoup](https://jsoup.org/) for parsing.
 
-XPath rules are supported, but the DOM is not a typical XML/XPath DOM. E.g.
-text nodes are normal nodes. This might change in the future.
+XPath 2.0 rules are supported, but the DOM is not always a typical XML/XPath DOM.
+In the Designer, text nodes appear as nodes with name "#text", but they can
+be selected as usual using `text()`.
+
+XML Namespaces are not supported. The local name of attributes include the prefix,
+so that you have to select attributes by e.g. `//*[@*[local-name() = 'if:true']]`.
+
+Only XPath 1.0 rules are not supported.

docs/pages/pmd/userdocs/cli_reference.md

@@ -76,14 +76,14 @@ The tool comes with a rather extensive help text, simply running with `--help`!
     %}
     {% include custom/cli_option_row.html options="--file-list"
                option_arg="filepath"
-               description="Path to file containing a comma delimited list of files to analyze.
+               description="Path to file containing a list of files to analyze, one path per line.
                             If this is given, then you don't need to provide `--dir`."
     %}
     {% include custom/cli_option_row.html options="--force-language"
                option_arg="lang"
                description="Force a language to be used for all input files, irrespective of
-                            filenames. When using this option, the automatic language selection
-                            by extension is disabled and all files are tried to be parsed with
+                            file names. When using this option, the automatic language selection
+                            by extension is disabled and PMD tries to parse all files with
                             the given language `<lang>`. Parsing errors are ignored and unparsable files
                             are skipped.
                             
@@ -92,9 +92,9 @@ The tool comes with a rather extensive help text, simply running with `--help`!
     %}
     {% include custom/cli_option_row.html options="--ignore-list"
                option_arg="filepath"
-               description="Path to file containing a comma delimited list of files to ignore.
+               description="Path to file containing a list of files to ignore, one path per line.
                             This option can be combined with `--dir` and `--file-list`.
-                            This ignore list takes precedence over any files in the filelist."
+                            This ignore list takes precedence over any files in the file-list."
     %}
     {% include custom/cli_option_row.html options="--help,-h,-H"
                description="Display help on usage."

docs/pages/pmd/userdocs/cpd/cpd.md

@@ -112,9 +112,9 @@ Novice as much as advanced readers may want to [read on on Refactoring Guru](htt
                languages="Java"
     %}
     {% include custom/cli_option_row.html options="--ignore-annotations"
-               description="Ignore language annotations when comparing text"
+               description="Ignore language annotations (Java) or attributes (C#) when comparing text"
                default="false"
-               languages="Java"
+               languages="C#, Java"
     %}
     {% include custom/cli_option_row.html options="--ignore-literal-sequences"
                description="Ignore sequences of literals (common e.g. in list initializers)"

docs/pages/release_notes.md

@@ -14,10 +14,52 @@ This is a {{ site.pmd.release_type }} release.
 
 ### New and noteworthy
 
+#### CLI improvements
+
+The PMD CLI now allows repeating the `--dir` (`-d`) and `--rulesets` (`-R`) options,
+ as well as providing several space-separated arguments to either of them. For instance:
+```shell
+pmd -d src/main/java src/test/java -R rset1.xml -R rset2.xml
+```
+This also allows globs to be used on the CLI if your shell supports shell expansion.
+For instance, the above can be written
+```shell
+pmd -d src/*/java -R rset*.xml
+```
+Please use theses new forms instead of using comma-separated lists as argument to these options.
+
+#### C# Improvements
+
+When executing CPD on C# sources, the option `--ignore-annotations` is now supported as well.
+It ignores C# attributes when detecting duplicated code. This option can also be enabled via
+the CPD GUI. See [#3974](https://github.com/pmd/pmd/pull/3974) for details.
+
 ### Fixed Issues
 
+* cli
+    * [#1445](https://github.com/pmd/pmd/issues/1445): \[core] Allow CLI to take globs as parameters
+* core
+    * [#3942](https://github.com/pmd/pmd/issues/3942): \[core] common-io path traversal vulnerability (CVE-2021-29425)
+* cs (c#)
+    * [#3974](https://github.com/pmd/pmd/pull/3974): \[cs] Add option to ignore C# attributes (annotations)
+* go
+    * [#2752](https://github.com/pmd/pmd/issues/2752): \[go] Error parsing unicode values
+* html
+    * [#3955](https://github.com/pmd/pmd/pull/3955): \[html] Improvements for handling text and comment nodes
+* java
+    * [#3423](https://github.com/pmd/pmd/issues/3423): \[java] Error processing identifiers with Unicode 
+* java-bestpractices
+    * [#3954](https://github.com/pmd/pmd/issues/3954): \[java] NPE in UseCollectionIsEmptyRule when .size() is called in a record
+* java-design
+    * [#3874](https://github.com/pmd/pmd/issues/3874): \[java] ImmutableField reports fields annotated with @Autowired (Spring) and @Mock (Mockito)
+* java-performance
+    * [#3379](https://github.com/pmd/pmd/issues/3379): \[java] UseArraysAsList must ignore primitive arrays
+    * [#3965](https://github.com/pmd/pmd/issues/3965): \[java] UseArraysAsList false positive with non-trivial loops
 * javascript
-  * [#3948](https://github.com/pmd/pmd/issues/3948): \[js] Invalid operator error for method property in object literal
+    * [#2605](https://github.com/pmd/pmd/issues/2605): \[js] Support unicode characters
+    * [#3948](https://github.com/pmd/pmd/issues/3948): \[js] Invalid operator error for method property in object literal
+* python
+    * [#2604](https://github.com/pmd/pmd/issues/2604): \[python] Support unicode identifiers
 
 ### API Changes
 
@@ -35,8 +77,15 @@ Use the explicit forms of these references to be compatible with PMD 7.
 - {% jdoc core::RuleSetReferenceId#toString() %} is now deprecated. The format of this
  method will remain the same until PMD 7. The deprecation is intended to steer users
  away from relying on this format, as it may be changed in PMD 7.
+- {% jdoc core::PMDConfiguration#getInputPaths() %} and
+{% jdoc core::PMDConfiguration#setInputPaths(java.lang.String) %} are now deprecated.
+A new set of methods have been added, which use lists and do not rely on comma splitting.
 
 ### External Contributions
 
+* [#3961](https://github.com/pmd/pmd/pull/3961): \[java] Fix #3954 - NPE in UseCollectionIsEmptyRule with record - [@flyhard](https://github.com/flyhard)
+* [#3964](https://github.com/pmd/pmd/pull/3964): \[java] Fix #3874 - ImmutableField: fix mockito/spring false positives - [@lukelukes](https://github.com/lukelukes)
+* [#3974](https://github.com/pmd/pmd/pull/3974): \[cs] Add option to ignore C# attributes (annotations) - [@maikelsteneker](https://github.com/maikelsteneker)
+
 {% endtocmaker %}
 

pmd-apex/pom.xml

@@ -58,10 +58,6 @@
             <type>pom</type>
         </dependency>
 
-        <dependency>
-            <groupId>commons-io</groupId>
-            <artifactId>commons-io</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>

pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexParser.java

@@ -8,12 +8,11 @@
 import java.io.Reader;
 import java.util.Map;
 
-import org.apache.commons.io.IOUtils;
-
 import net.sourceforge.pmd.annotation.InternalApi;
 import net.sourceforge.pmd.lang.apex.ApexJorjeLogging;
 import net.sourceforge.pmd.lang.apex.ApexParserOptions;
 import net.sourceforge.pmd.lang.ast.ParseException;
+import net.sourceforge.pmd.util.IOUtil;
 
 import apex.jorje.data.Locations;
 import apex.jorje.semantic.ast.compilation.Compilation;
@@ -50,7 +49,7 @@ public Compilation parseApex(final String sourceCode) throws ParseException {
 
     public ApexNode<Compilation> parse(final Reader reader) {
         try {
-            final String sourceCode = IOUtils.toString(reader);
+            final String sourceCode = IOUtil.readToString(reader);
             final Compilation astRoot = parseApex(sourceCode);
             final ApexTreeBuilder treeBuilder = new ApexTreeBuilder(sourceCode, parserOptions);
             suppressMap = treeBuilder.getSuppressMap();

pmd-core/pom.xml

@@ -111,10 +111,6 @@
             <groupId>com.beust</groupId>
             <artifactId>jcommander</artifactId>
         </dependency>
-        <dependency>
-            <groupId>commons-io</groupId>
-            <artifactId>commons-io</artifactId>
-        </dependency>
         <dependency>
             <groupId>jaxen</groupId>
             <artifactId>jaxen</artifactId>

pmd-core/src/main/java/net/sourceforge/pmd/PMDConfiguration.java

@@ -8,6 +8,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Properties;
@@ -102,7 +103,7 @@ public class PMDConfiguration extends AbstractConfiguration {
     // Rule and source file options
     private List<String> ruleSets = new ArrayList<>();
     private RulePriority minimumPriority = RulePriority.LOW;
-    private String inputPaths;
+    private List<String> inputPaths = new ArrayList<>();
     private String inputUri;
     private String inputFilePath;
     private String ignoreFilePath;
@@ -420,19 +421,54 @@ public void setMinimumPriority(RulePriority minimumPriority) {
      * Get the comma separated list of input paths to process for source files.
      *
      * @return A comma separated list.
+     *
+     * @deprecated Use {@link #getAllInputPaths()}
      */
+    @Deprecated
     public String getInputPaths() {
-        return inputPaths;
+        return inputPaths.isEmpty() ? null : StringUtils.join(inputPaths, ",");
+    }
+
+    /**
+     * Returns an unmodifiable list.
+     *
+     * @throws NullPointerException If the parameter is null
+     */
+    public List<String> getAllInputPaths() {
+        return Collections.unmodifiableList(inputPaths);
     }
 
     /**
      * Set the comma separated list of input paths to process for source files.
      *
-     * @param inputPaths
-     *            The comma separated list.
+     * @param inputPaths The comma separated list.
+     *
+     * @throws NullPointerException If the parameter is null
+     * @deprecated Use {@link #setInputPaths(List)} or {@link #addInputPath(String)}
      */
+    @Deprecated
     public void setInputPaths(String inputPaths) {
-        this.inputPaths = inputPaths;
+        List<String> paths = new ArrayList<>();
+        Collections.addAll(paths, inputPaths.split(","));
+        this.inputPaths = paths;
+    }
+
+    /**
+     * Set the input paths to the given list of paths.
+     * @throws NullPointerException If the parameter is null
+     */
+    public void setInputPaths(List<String> inputPaths) {
+        this.inputPaths = new ArrayList<>(inputPaths);
+    }
+
+    /**
+     * Add an input path. It is not split on commas.
+     *
+     * @throws NullPointerException If the parameter is null
+     */
+    public void addInputPath(String inputPath) {
+        Objects.requireNonNull(inputPath);
+        this.inputPaths.add(inputPath);
     }
 
     public String getInputFilePath() {
@@ -526,7 +562,7 @@ public Renderer createRenderer(boolean withReportWriter) {
         Renderer renderer = RendererFactory.createRenderer(reportFormat, reportProperties);
         renderer.setShowSuppressedViolations(showSuppressedViolations);
         if (reportShortNames && inputPaths != null) {
-            renderer.setUseShortNames(Arrays.asList(inputPaths.split(",")));
+            renderer.setUseShortNames(Collections.unmodifiableList(new ArrayList<>(inputPaths)));
         }
         if (withReportWriter) {
             renderer.setReportFile(reportFile);

pmd-core/src/main/java/net/sourceforge/pmd/RuleSetFactoryCompatibility.java

@@ -16,9 +16,8 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.commons.io.IOUtils;
-
 import net.sourceforge.pmd.annotation.InternalApi;
+import net.sourceforge.pmd.util.IOUtil;
 
 /**
  * Provides a simple filter mechanism to avoid failing to parse an old ruleset,
@@ -116,7 +115,7 @@ void addFilterRuleRemoved(String language, String ruleset, String name) {
      * @throws IOException if the stream couldn't be read
      */
     public Reader filterRuleSetFile(InputStream stream) throws IOException {
-        byte[] bytes = IOUtils.toByteArray(stream);
+        byte[] bytes = IOUtil.toByteArray(stream);
         String encoding = determineEncoding(bytes);
         String ruleset = new String(bytes, encoding);
 

pmd-core/src/main/java/net/sourceforge/pmd/RuleSetWriter.java

@@ -22,7 +22,6 @@
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
 
-import org.apache.commons.io.IOUtils;
 import org.w3c.dom.CDATASection;
 import org.w3c.dom.DOMException;
 import org.w3c.dom.Document;
@@ -37,6 +36,7 @@
 import net.sourceforge.pmd.properties.PropertyDescriptor;
 import net.sourceforge.pmd.properties.PropertyDescriptorField;
 import net.sourceforge.pmd.properties.PropertyTypeId;
+import net.sourceforge.pmd.util.IOUtil;
 
 /**
  * This class represents a way to serialize a RuleSet to an XML configuration
@@ -62,7 +62,7 @@ public RuleSetWriter(OutputStream outputStream) {
     }
 
     public void close() {
-        IOUtils.closeQuietly(outputStream);
+        IOUtil.closeQuietly(outputStream);
     }
 
     public void write(RuleSet ruleSet) {

pmd-core/src/main/java/net/sourceforge/pmd/ant/Formatter.java

@@ -18,7 +18,6 @@
 import java.util.List;
 import java.util.Properties;
 
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.apache.commons.lang3.reflect.MethodUtils;
@@ -30,6 +29,7 @@
 import net.sourceforge.pmd.annotation.InternalApi;
 import net.sourceforge.pmd.renderers.Renderer;
 import net.sourceforge.pmd.renderers.RendererFactory;
+import net.sourceforge.pmd.util.IOUtil;
 
 public class Formatter {
 
@@ -187,8 +187,8 @@ private static Writer getToFileWriter(String baseDir, File toFile, Charset chars
             isOnError = false;
         } finally {
             if (isOnError) {
-                IOUtils.closeQuietly(output);
-                IOUtils.closeQuietly(writer);
+                IOUtil.closeQuietly(output);
+                IOUtil.closeQuietly(writer);
             }
         }
         return writer;