Merge pull request #3861 from adangel:regression-tester-update

[ci] Add apex projects for regression testing #3861
pmd · Mar 27, 2022 · 43ee8df · 43ee8df
2 parents 522605d + 0081e61
commit 43ee8df
Show file tree

Hide file tree

Showing 9 changed files with 134 additions and 52 deletions.
diff --git a/.ci/files/all-java.xml → .ci/files/all-regression-rules.xml b/.ci/files/all-java.xml → .ci/files/all-regression-rules.xml
@@ -1,10 +1,19 @@
 <?xml version="1.0"?>
 
-<ruleset name="All Java Rules" 
+<ruleset name="All Regression Rules" 
     xmlns="http://pmd.sourceforge.net/ruleset/2.0.0"
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:schemaLocation="http://pmd.sourceforge.net/ruleset/2.0.0 https://pmd.sourceforge.io/ruleset_2_0_0.xsd">
-    <description>Every java rule in PMD which is used for the regression tests with pmdtester</description>
+    <description>Every apex and java rule in PMD which is used for the regression tests with pmdtester</description>
+
+    <rule ref="category/apex/bestpractices.xml" />
+    <rule ref="category/apex/codestyle.xml" />
+    <rule ref="category/apex/design.xml" />
+    <rule ref="category/apex/documentation.xml" />
+    <rule ref="category/apex/errorprone.xml" />
+    <rule ref="category/apex/multithreading.xml" />
+    <rule ref="category/apex/performance.xml" />
+    <rule ref="category/apex/security.xml" />
 
     <rule ref="category/java/bestpractices.xml" />
     <rule ref="category/java/codestyle.xml" />

diff --git a/.ci/files/project-list.xml b/.ci/files/project-list.xml
@@ -146,4 +146,34 @@ fi
     <tag>jdk-11+28</tag>
     <src-subpath>src/java.base</src-subpath>
   </project>
+
+  <project>
+    <name>Schedul-o-matic-9000</name>
+    <type>git</type>
+    <connection>https://github.com/pmd/Schedul-o-matic-9000</connection>
+    <!-- 
+    <connection>https://github.com/SalesforceLabs/Schedul-o-matic-9000</connection>
+    Note: using forked repo from pmd, as original repo doesn't have any useful tags/branches
+    -->
+    <tag>pmd-regression-test</tag>
+  </project>
+
+  <project>
+    <name>fflib-apex-common</name>
+    <type>git</type>
+    <connection>https://github.com/pmd/fflib-apex-common</connection>
+    <!-- 
+    <connection>https://github.com/apex-enterprise-patterns/fflib-apex-common</connection>
+    Note: using forked repo from pmd, as original repo doesn't have any useful tags/branches
+     -->
+    <tag>pmd-regression-test</tag>
+  </project>
+
+  <project>
+    <name>apex-link</name>
+    <type>git</type>
+    <connection>https://github.com/nawforce/apex-link</connection>
+    <tag>v2.3.0</tag>
+    <src-subpath>samples</src-subpath>
+  </project>
 </projectlist>
diff --git a/.ci/inc/regression-tester.inc b/.ci/inc/regression-tester.inc
@@ -48,7 +48,7 @@ function regression_tester_uploadBaseline() {
         --mode single \
         --local-git-repo ./pmd \
         --patch-branch "${baseline_branch}" \
-        --patch-config ./pmd/.ci/files/all-java.xml \
+        --patch-config ./pmd/.ci/files/all-regression-rules.xml \
         --list-of-project ./pmd/.ci/files/project-list.xml --html-flag \
         --error-recovery
     pushd target/reports || { echo "Directory 'target/reports' doesn't exist"; exit 1; }

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -33,9 +33,9 @@ jobs:
           ~/.cache
           ~/work/pmd/target/repositories
           vendor/bundle
-        key: v2-${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
+        key: v3-${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
         restore-keys: |
-          v2-${{ runner.os }}-
+          v3-${{ runner.os }}-
     - name: Set up Ruby 2.7
       uses: ruby/setup-ruby@v1
       with:

diff --git a/.github/workflows/troubleshooting.yml b/.github/workflows/troubleshooting.yml
@@ -21,9 +21,9 @@ jobs:
               ~/.cache
               ~/work/pmd/target/repositories
               vendor/bundle
-          key: v2-${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
+          key: v3-${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            v2-${{ runner.os }}-
+            v3-${{ runner.os }}-
       - name: Set up Ruby 2.7
         uses: ruby/setup-ruby@v1
         with:

diff --git a/Dangerfile b/Dangerfile
@@ -1,61 +1,103 @@
 require 'pmdtester'
 require 'time'
 require 'logger'
+require 'fileutils'
 
 @logger = Logger.new(STDOUT)
 
+def get_args(base_branch, autogen = TRUE, patch_config = './pmd/.ci/files/all-regression-rules.xml')
+  ['--local-git-repo', './pmd',
+   '--list-of-project', './pmd/.ci/files/project-list.xml',
+   '--base-branch', base_branch,
+   '--patch-branch', 'HEAD',
+   '--patch-config', patch_config,
+   '--mode', 'online',
+   autogen ? '--auto-gen-config' : '--filter-with-patch-config',
+   '--keep-reports',
+   '--error-recovery',
+   '--baseline-download-url', 'https://pmd-code.org/pmd-regression-tester/',
+   # '--debug',
+   ]
+end
+
 def run_pmdtester
   Dir.chdir('..') do
-    branch_name = "#{ENV['PMD_CI_BRANCH']}"
-    argv = ['--local-git-repo', './pmd',
-            '--list-of-project', './pmd/.ci/files/project-list.xml',
-            '--base-branch', branch_name,
-            '--patch-branch', 'HEAD',
-            '--patch-config', './pmd/.ci/files/all-java.xml',
-            '--mode', 'online',
-            '--auto-gen-config',
-            '--error-recovery',
-            '--baseline-download-url', 'https://pmd-code.org/pmd-regression-tester/',
-            #'--debug',
-            ]
     begin
-      @summary = PmdTester::Runner.new(argv).run
-      upload_report
+      @base_branch = ENV['PMD_CI_BRANCH']
+      @logger.info "\n\n--------------------------------------"
+      @logger.info "Run against PR base #{@base_branch}"
+      @summary = PmdTester::Runner.new(get_args(@base_branch)).run
+
+      unless Dir.exist?('target/reports/diff')
+        message("No regression tested rules have been changed.", sticky: true)
+        return
+      end
+
+      # move the generated report out of the way
+      FileUtils.mv 'target/reports/diff', 'target/diff1'
+      message1 = create_message
+
+      # run against master branch (if the PR is not already against master)
+      unless ENV['PMD_CI_BRANCH'] == 'master'
+        @base_branch = 'master'
+        @logger.info "\n\n--------------------------------------"
+        @logger.info "Run against #{@base_branch}"
+        @summary = PmdTester::Runner.new(get_args(@base_branch)).run
+
+        # move the generated report out of the way
+        FileUtils.mv 'target/reports/diff', 'target/diff2'
+        message2 = create_message
+      end
+
+      report_url = upload_report
+
+      if report_url
+        message1 += "[Full report](#{report_url}/diff1/index.html)"
+        # set value of sticky to true and the message is kept after new commits are submitted to the PR
+        message(message1, sticky: true)
+
+        if message2
+          message2 += "[Full report](#{report_url}/diff2/index.html)"
+          # set value of sticky to true and the message is kept after new commits are submitted to the PR
+          message(message2, sticky: true)
+        end
+      end
+
     rescue StandardError => e
       warn("Running pmdtester failed, this message is mainly used to remind the maintainers of PMD.")
       @logger.error "Running pmdtester failed: #{e.inspect}"
     end
   end
 end
 
+def create_message
+  "Compared to #{@base_branch}:\n"\
+  "This changeset " \
+  "changes #{@summary[:violations][:changed]} violations,\n" \
+  "introduces #{@summary[:violations][:new]} new violations, " \
+  "#{@summary[:errors][:new]} new errors and " \
+  "#{@summary[:configerrors][:new]} new configuration errors,\n" \
+  "removes #{@summary[:violations][:removed]} violations, "\
+  "#{@summary[:errors][:removed]} errors and " \
+  "#{@summary[:configerrors][:removed]} configuration errors.\n"
+end
+
 def upload_report
-  Dir.chdir('target/reports') do
+  Dir.chdir('target') do
     tar_filename = "pr-#{ENV['PMD_CI_PULL_REQUEST_NUMBER']}-diff-report-#{Time.now.strftime("%Y-%m-%dT%H-%M-%SZ")}.tar.gz"
-    unless Dir.exist?('diff/')
-      message("No java rules are changed!", sticky: true)
-      return
-    end
 
-    `tar czf #{tar_filename} diff/`
+    `tar czf #{tar_filename} diff1/ diff2/`
     tar_size = (10 * File.size(tar_filename) / 1024 / 1024)/10.0
     @logger.info "Uploading file #{tar_filename} (#{tar_size}mb) now..."
     report_url = `curl -u #{ENV['PMD_CI_CHUNK_TOKEN']} -T #{tar_filename} https://chunk.io`
     if $?.success?
+      report_url.chomp!
       @logger.info "Successfully uploaded #{tar_filename} to #{report_url}"
-
-      # set value of sticky to true and the message is kept after new commits are submitted to the PR
-      message("This changeset " \
-              "changes #{@summary[:violations][:changed]} violations,\n" \
-              "introduces #{@summary[:violations][:new]} new violations, " \
-              "#{@summary[:errors][:new]} new errors and " \
-              "#{@summary[:configerrors][:new]} new configuration errors,\n" \
-              "removes #{@summary[:violations][:removed]} violations, "\
-              "#{@summary[:errors][:removed]} errors and " \
-              "#{@summary[:configerrors][:removed]} configuration errors.\n" \
-              "[Full report](#{report_url.chomp}/diff/index.html)", sticky: true)
+      report_url
     else
       @logger.error "Error while uploading #{tar_filename} to chunk.io: #{report_url}"
       warn("Uploading the diff report failed, this message is mainly used to remind the maintainers of PMD.")
+      nil
     end
   end
 end

diff --git a/Gemfile b/Gemfile
@@ -10,9 +10,9 @@ gem 'danger'
 # this happens during release (.ci/build.sh and do-release.sh)
 # but also during regular builds (.ci/build.sh)
 group :release_notes_preprocessing do
-  gem 'liquid', '>=4.0.0'
-  gem 'safe_yaml', '>=1.0'
-  gem 'rouge', '>= 1.7', '< 4'
+  gem 'liquid'
+  gem 'safe_yaml'
+  gem 'rouge'
 end
 
 # vim: syntax=ruby
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -9,10 +9,10 @@ GEM
       nap
       open4 (~> 1.3)
     colored2 (3.1.2)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.1.10)
     cork (0.3.0)
       colored2 (~> 3.1)
-    danger (8.4.5)
+    danger (8.5.0)
       claide (~> 1.0)
       claide-plugins (>= 0.9.2)
       colored2 (~> 3.1)
@@ -58,11 +58,11 @@ GEM
       raabro (~> 1.4)
     git (1.10.2)
       rchardet (~> 1.8)
-    kramdown (2.3.1)
+    kramdown (2.3.2)
       rexml
     kramdown-parser-gfm (1.1.0)
       kramdown (~> 2.0)
-    liquid (5.2.0)
+    liquid (5.3.0)
     logger-colors (1.0.0)
     mini_portile2 (2.8.0)
     multipart-post (2.1.1)
@@ -75,12 +75,12 @@ GEM
       faraday (>= 0.9)
       sawyer (~> 0.8.0, >= 0.5.3)
     open4 (1.3.4)
-    pmdtester (1.3.0)
+    pmdtester (1.4.0)
       differ (~> 0.1)
-      liquid (>= 4.0)
+      liquid (~> 5.2)
       logger-colors (~> 1.0)
-      nokogiri (>= 1.11.0.rc4)
-      rufus-scheduler (~> 3.5)
+      nokogiri (~> 1.13)
+      rufus-scheduler (~> 3.8)
       slop (~> 4.6)
     public_suffix (4.0.6)
     raabro (1.4.0)
@@ -107,10 +107,10 @@ PLATFORMS
 
 DEPENDENCIES
   danger
-  liquid (>= 4.0.0)
+  liquid
   pmdtester
-  rouge (>= 1.7, < 4)
-  safe_yaml (>= 1.0)
+  rouge
+  safe_yaml
 
 BUNDLED WITH
    2.1.4
diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
@@ -101,6 +101,7 @@ The CLI itself remains compatible, if you run PMD via command-line, no action is
 *   apex-performance
     *   [#3773](https://github.com/pmd/pmd/pull/3773): \[apex] EagerlyLoadedDescribeSObjectResult false positives with SObjectField.getDescribe()
 *   core
+    *   [#2693](https://github.com/pmd/pmd/issues/2693): \[ci] Add integration tests with real open-source projects
     *   [#3299](https://github.com/pmd/pmd/issues/3299): \[core] Deprecate system properties of PMDCommandLineInterface
 *   doc
     *   [#2504](https://github.com/pmd/pmd/issues/2504): \[doc] Improve "Edit me on github" button