[CI] Enable compatibility test matrix for unit tests

.ci/jobs/apm-agent-java-schedule-weekly.yml

@@ -0,0 +1,27 @@
+---
+- job:
+    name: apm-agent-java/apm-schedule-weekly
+    display-name: Jobs scheduled weekly (Monday)
+    description: Jobs scheduled weekly (Monday)
+    view: APM-CI
+    project-type: pipeline
+    parameters:
+      - string:
+          name: branch_specifier
+          default: master
+          description: the Git branch specifier to build
+    pipeline-scm:
+      script-path: .ci/schedule-weekly.groovy
+      scm:
+        - git:
+            url: git@github.com:elastic/apm-agent-java.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/apm-agent-java.git
+            branches:
+              - $branch_specifier
+    triggers:
+      - timed: 'H H(1-4) * * 1'

.ci/schedule-weekly.groovy

@@ -0,0 +1,39 @@
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    NOTIFY_TO = credentials('notify-to')
+    PIPELINE_LOG_LEVEL='INFO'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+  }
+  triggers {
+    cron('H H(1-4) * * 1')
+  }
+  stages {
+    stage('Run compatibility tests') {
+      steps {
+        build(job: 'apm-agent-java/apm-agent-java-mbp/master',
+          parameters: [
+            booleanParam(name: 'Run_As_Master_Branch', value: true),
+            booleanParam(name: 'compatibility_ci', value: true)
+          ],
+          propagate: false,
+          wait: false
+        )
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult()
+    }
+  }
+}

Jenkinsfile

@@ -28,7 +28,7 @@ pipeline {
     quietPeriod(10)
   }
   triggers {
-    issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?(?:benchmark\\W+)?tests(?:\\W+please)?|/test).*')
+    issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?(?:(compatibility|benchmark)\\W+)?tests(?:\\W+please)?.*|^/test(?:\\W+.*)?$)')
   }
   parameters {
     string(name: 'MAVEN_CONFIG', defaultValue: '-V -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dhttps.protocols=TLSv1.2 -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.httpconnectionManager.ttlSeconds=25', description: 'Additional maven options.')
@@ -37,8 +37,8 @@ pipeline {
     booleanParam(name: 'smoketests_ci', defaultValue: true, description: 'Enable Smoke tests')
     booleanParam(name: 'bench_ci', defaultValue: true, description: 'Enable benchmarks')
     booleanParam(name: 'push_docker', defaultValue: false, description: 'Push Docker image during release stage')
+    booleanParam(name: 'compatibility_ci', defaultValue: false, description: 'Enable compatibility tests')
   }
-
   stages {
     stage('Initializing'){
       options { skipDefaultCheckout() }
@@ -305,6 +305,50 @@ pipeline {
         githubNotify(context: "${env.GITHUB_CHECK_ITS_NAME}", description: "${env.GITHUB_CHECK_ITS_NAME} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${env.ITS_PIPELINE.replaceAll('/','+')}")
       }
     }
+    stage('Unit Tests') {
+      options { skipDefaultCheckout() }
+      when {
+        beforeAgent true
+        anyOf {
+          not { changeRequest() }
+          expression { return params.compatibility_ci }
+          expression { return env.GITHUB_COMMENT?.contains('compatibility tests') }
+        }
+      }
+      matrix {
+        agent { label 'linux && immutable' }
+        axes {
+          axis {
+            // the list of support java versions can be found in the infra repo (ansible/roles/java/defaults/main.yml)
+            name 'JAVA_VERSION'
+            values 'java12', 'openjdk12', 'openjdk13', 'openjdk14', 'openjdk15', 'openjdk16'
+          }
+        }
+        stages {
+          stage('Test') {
+            environment {
+              HOME = "${env.WORKSPACE}"
+              JAVA_HOME = "${env.HUDSON_HOME}/.java/${JAVA_VERSION}"
+              PATH = "${env.JAVA_HOME}/bin:${env.PATH}"
+            }
+            steps {
+              withGithubNotify(context: "Unit Tests ${JAVA_VERSION}", tab: 'tests') {
+                deleteDir()
+                unstash 'build'
+                dir("${BASE_DIR}"){
+                  sh(label: "./mvnw test for ${JAVA_VERSION}", script: './mvnw test')
+                }
+              }
+            }
+            post {
+              always {
+                reportTestResults()
+              }
+            }
+          }
+        }
+      }
+    }
     stage('Stable') {
       options { skipDefaultCheckout() }
       when {