Merge pull request #7860 from AdamBrousseau/jitserver_spec

Add JITServer spec to pipelines
eclipse-openj9 · Dec 4, 2019 · 78476fc · 78476fc
2 parents af07809 + d366f09
commit 78476fc
Show file tree

Hide file tree

Showing 5 changed files with 199 additions and 11 deletions.
diff --git a/buildenv/jenkins/README.md b/buildenv/jenkins/README.md
@@ -39,6 +39,9 @@ This folder contains Jenkins pipeline scripts that are used in the OpenJ9 Jenkin
     - Linux on x86-64 with CMake
         - Spec: x86-64_linux_cm
         - Shortname: xlinuxcm or xlinuxcmake
+    - Linux on x86-64 with JITServer
+        - Spec x86-64_linux_jit
+        - Shortname: xlinuxjit
     - Linux on s390x largeheap/non-compressed references
         - Spec: s390x_linux_xl
         - Shortname: zlinuxlargeheap or zlinuxxl
@@ -89,7 +92,7 @@ This folder contains Jenkins pipeline scripts that are used in the OpenJ9 Jenkin
     - `<version>` is the number of the supported release, e.g. 8 | 11 | next
 - Note: You can use keyword `all` for platform but not for test level/type or JDK versions.
 - Note: For backward compatibility `<level>.<test type>` equal to `sanity` or `extended` is acceptable and will map to `sanity.functional` and `extended.functional` respectively.
-- Note: TEST_FLAG is an optional argument to the test target. It is recommended to only launch 1 test target with 1 TEST_FLAG per comment/build. At the time of writing, the two supported test flags are `+jitaas` and `+aot`. Ex. `Jenkins test sanity.functional+jitaas xlinux jdk8`.
+- Note: TEST_FLAG is an optional argument to the test target. It is recommended to only launch 1 test target with 1 TEST_FLAG per comment/build. At the time of writing, the two supported test flags are `+jitaas` and `+aot`. Ex. `Jenkins test sanity.functional+jitaas xlinux jdk8`. Also note JITServer specs set the TEST_FLAG via the variable file so it is unnecessary to add it to the test target.
 
 ###### Examples
 - Request a Compile-only build on all platforms and multiple versions by commenting in a PR

diff --git a/buildenv/jenkins/common/pipeline-functions.groovy b/buildenv/jenkins/common/pipeline-functions.groovy
@@ -425,7 +425,12 @@ def workflow(SDK_VERSION, SPEC, SHAS, OPENJDK_REPO, OPENJDK_BRANCH, OPENJ9_REPO,
                 echo "The '${name}' test suite will be excluded"
                 continue
             }
-            def TEST_FLAG = (name.contains('+')) ? name.substring(name.indexOf('+')+1).toUpperCase() : ''
+            // Add a TEST_FLAG to a specific target if one was passed with the test target. Eg. sanity.functional+aot
+            if (name.contains('+')) {
+                def temp_test_flag = name.substring(name.indexOf('+')+1).toUpperCase()
+                TEST_FLAG = (TEST_FLAG) ? TEST_FLAG + ',' + temp_test_flag : temp_test_flag
+            }
+            echo "TEST_FLAG:'${TEST_FLAG}'"
             def TEST_JOB_NAME = get_test_job_name(get_target_name(name), SPEC, SDK_VERSION, BUILD_IDENTIFIER)
 
             def IS_PARALLEL = false
@@ -493,7 +498,7 @@ def get_build_job_name(spec, version, identifier) {
 }
 
 def get_test_job_name(targetName, spec, version, identifier) {
-    spec = strip_cmake_specs(spec)
+    spec = strip_spec_suffixes(spec)
     id = convert_build_identifier(identifier)
     return "Test_openjdk${version}_j9_${targetName}_${spec}_${id}"
 }
@@ -614,7 +619,7 @@ def generate_test_jobs(TARGET_NAMES, SPEC, ARTIFACTORY_SERVER, ARTIFACTORY_REPO)
     levels.unique(true)
     groups.unique(true)
 
-    spec = strip_cmake_specs(SPEC)
+    spec = strip_spec_suffixes(SPEC)
     if (levels && groups) {
         def parameters = [
             string(name: 'LEVELS', value: levels.join(',')),
@@ -768,11 +773,12 @@ def setup_pull_request_single_comment(PARSED_COMMENT) {
     PERSONAL_BUILD = 'true'
 }
 
-def strip_cmake_specs(spec) {
-    // Strip off cmake from SPEC name
-    // Since cmake is a method for how the SDK is built,
+def strip_spec_suffixes (spec) {
+    // Strip off cmake/jit from SPEC name
+    // Since cmake and JITServer is a method for how the SDK is built,
     // from a test perspective it is the same sdk. Therefore
-    // we will reuse the same test jobs as the non-cmake specs.
-    return spec - '_cm'
+    // we will reuse the same test jobs as the regular specs.
+    return spec - '_cm' - '_jit'
 }
+
 return this
diff --git a/buildenv/jenkins/common/variables-functions.groovy b/buildenv/jenkins/common/variables-functions.groovy
@@ -453,7 +453,8 @@ def get_date() {
 }
 
 /*
-* Set TESTS_TARGETS, indicating the level of testing.
+* Set TESTS_TARGETS, indicating the level of testing
+* and sets TEST_FLAG for all targets if defined in variable file
 */
 def set_test_targets() {
     TESTS_TARGETS = params.TESTS_TARGETS
@@ -470,8 +471,14 @@ def set_test_targets() {
         }
     }
 
+    TEST_FLAG = ''
+    if (VARIABLES."${SPEC}".test_flags) {
+        TEST_FLAG = get_value(VARIABLES."${SPEC}".test_flags, SDK_VERSION)
+    }
+
     echo "TESTS_TARGETS: ${TESTS_TARGETS}"
     echo "EXCLUDED_TESTS: ${EXCLUDED_TESTS}"
+    echo "TEST_FLAG: ${TEST_FLAG}"
 }
 
 def get_default_test_targets() {

diff --git a/buildenv/jenkins/jobs/pipelines/Pipeline-Build-Test-All.groovy b/buildenv/jenkins/jobs/pipelines/Pipeline-Build-Test-All.groovy
@@ -83,13 +83,16 @@ CURRENT_RELEASES = ['8', '11', '13', 'next']
 
 SPECS = ['ppc64_aix'      : CURRENT_RELEASES,
          'ppc64le_linux'  : CURRENT_RELEASES,
+         'ppc64le_linux_jit' : ['8', '11'],
          'ppc64le_linux_xl' : CURRENT_RELEASES,
          's390x_linux'    : CURRENT_RELEASES,
+         's390x_linux_jit' : ['8', '11'],
          's390x_linux_xl' : CURRENT_RELEASES,
          's390x_zos'      : ['11'],
          'x86-64_linux_xl': CURRENT_RELEASES,
          'x86-64_linux'   : CURRENT_RELEASES,
          'x86-64_linux_cm': ['8', '11'],
+         'x86-64_linux_jit' : ['8', '11'],
          'x86-64_mac_xl'  : CURRENT_RELEASES,
          'x86-64_mac'     : CURRENT_RELEASES,
          'x86-32_windows' : ['8'],
@@ -103,16 +106,19 @@ SPECS = ['ppc64_aix'      : CURRENT_RELEASES,
 SHORT_NAMES = ['all' : ['ppc64le_linux','ppc64le_linux_xl','s390x_linux','s390x_linux_xl','x86-64_linux','x86-64_linux_xl','ppc64_aix','x86-64_windows','x86-32_windows','x86-64_mac'],
             'aix' : ['ppc64_aix'],
             'zlinux' : ['s390x_linux'],
+            'zlinuxjit' : ['s390x_linux_jit'],
             'zlinuxlargeheap' : ['s390x_linux_xl'],
             'zlinuxxl' : ['s390x_linux_xl'],
             'plinux' : ['ppc64le_linux'],
+            'plinuxjit' : ['ppc64le_linux_jit'],
             'plinuxlargeheap' : ['ppc64le_linux_xl'],
             'plinuxxl' : ['ppc64le_linux_xl'],
             'xlinuxlargeheap' : ['x86-64_linux_xl'],
             'xlinuxxl' : ['x86-64_linux_xl'],
             'xlinux' : ['x86-64_linux'],
             'xlinuxcmake' : ['x86-64_linux_cm'],
             'xlinuxcm' : ['x86-64_linux_cm'],
+            'xlinuxjit' : ['x86-64_linux_jit'],
             'win32' : ['x86-32_windows'],
             'win' : ['x86-64_windows'],
             'winlargeheap' : ['x86-64_windows_xl'],
@@ -540,7 +546,6 @@ def get_summary_table(identifier) {
 
                 innerTable += "<tr style=\"vertical-align: bottom;\"><td>${aLabel}</td><td style=\"text-align: right\">${link}</td><td style=\"text-align: right\">${duration}</td></tr>"
             }
-
 
             innerTable += "</tbody></table>"
             summaryText += "<td>${innerTable}</td>"

diff --git a/buildenv/jenkins/variables/defaults.yml b/buildenv/jenkins/variables/defaults.yml
@@ -194,6 +194,59 @@ ppc64le_linux_xl:
     13:
       ? special.system
 #========================================#
+# Linux PPCLE 64bits Compressed Pointers /w JITSERVER
+#========================================#
+ppc64le_linux_jit:
+  boot_jdk:
+    8: '/usr/lib/jvm/adoptojdk-java-80'
+    11: '/usr/lib/jvm/adoptojdk-java-11'
+    12: '/usr/lib/jvm/adoptojdk-java-11'
+    13: '/usr/lib/jvm/adoptojdk-java-12'
+    next: '/usr/lib/jvm/adoptojdk-java-13'
+  release:
+    8: 'linux-ppc64le-normal-server-release'
+    11: 'linux-ppc64le-normal-server-release'
+    12: 'linux-ppc64le-server-release'
+    13: 'linux-ppc64le-server-release'
+    next: 'linux-ppc64le-server-release'
+  freemarker: '/home/jenkins/freemarker.jar'
+  openjdk_reference_repo: '/home/jenkins/openjdk_cache'
+  node_labels:
+    build:
+      8: 'ci.role.build && hw.arch.ppc64le && sw.os.ubuntu'
+      11: 'ci.role.build && hw.arch.ppc64le && sw.os.ubuntu'
+      12: 'ci.role.build && hw.arch.ppc64le && sw.os.ubuntu'
+      13: 'ci.role.build && hw.arch.ppc64le && sw.os.ubuntu'
+      next: 'ci.role.build && hw.arch.ppc64le && sw.os.ubuntu'
+  extra_getsource_options:
+    8: '--openssl-version=1.1.1d'
+    11: '--openssl-version=1.1.1d'
+    12: '--openssl-version=1.1.1d'
+    13: '--openssl-version=1.1.1d'
+    next: '--openssl-version=1.1.1d'
+  extra_configure_options:
+    8: '--enable-cuda --with-cuda=/usr/local/cuda-9.0 --with-openssl=fetched --enable-jitserver=yes'
+    11: '--enable-cuda --with-cuda=/usr/local/cuda-9.0 --with-openssl=fetched --enable-jitserver=yes'
+    12: '--enable-cuda --with-cuda=/usr/local/cuda-9.0 --with-openssl=fetched --enable-jitserver=yes'
+    13: '--enable-cuda --with-cuda=/usr/local/cuda-9.0 --with-openssl=fetched --enable-jitserver=yes'
+    next: '--enable-cuda --with-cuda=/usr/local/cuda-9.0 --with-openssl=fetched --enable-jitserver=yes'
+  build_env:
+    vars:
+      8: 'CC=gcc-7 CXX=g++-7'
+      11: 'CC=gcc-7 CXX=g++-7'
+      12: 'CC=gcc-7 CXX=g++-7'
+      13: 'CC=gcc-7 CXX=g++-7'
+      next: 'CC=gcc-7 CXX=g++-7'
+  excluded_tests:
+    13:
+      ? special.system
+  test_flags:
+    8: 'JITAAS'
+    11: 'JITAAS'
+    12: 'JITAAS'
+    13: 'JITAAS'
+    next: 'JITAAS'
+#========================================#
 # Linux S390 64bits Compressed Pointers
 # Note: boot_jdk 8 must use an Adopt JDK8 build rather than an
 # IBM 7 for the bootJDK or compiling corba will fail to find Object.
@@ -298,6 +351,63 @@ s390x_linux_xl:
     13:
       ? special.system
 #========================================#
+# Linux S390 64bits Compressed Pointers /w JITSERVER
+# Note: boot_jdk 8 must use an Adopt JDK8 build rather than an
+# IBM 7 for the bootJDK or compiling corba will fail to find Object.
+#========================================#
+s390x_linux_jit:
+  boot_jdk:
+    8: '/usr/lib/jvm/adoptojdk-java-s390x-80'
+    11: '/usr/lib/jvm/adoptojdk-java-11'
+    12: '/usr/lib/jvm/adoptojdk-java-11'
+    13: '/usr/lib/jvm/adoptojdk-java-12'
+    next: '/usr/lib/jvm/adoptojdk-java-13'
+  release:
+    8: 'linux-s390x-normal-server-release'
+    11: 'linux-s390x-normal-server-release'
+    12: 'linux-s390x-server-release'
+    13: 'linux-s390x-server-release'
+    next: 'linux-s390x-server-release'
+  freemarker: '/home/jenkins/freemarker.jar'
+  openjdk_reference_repo: '/home/jenkins/openjdk_cache'
+  node_labels:
+    build:
+      8: 'ci.role.build && hw.arch.s390x && sw.os.ubuntu'
+      11: 'ci.role.build && hw.arch.s390x && sw.os.ubuntu'
+      12: 'ci.role.build && hw.arch.s390x && sw.os.ubuntu'
+      13: 'ci.role.build && hw.arch.s390x && sw.os.ubuntu'
+      next: 'ci.role.build && hw.arch.s390x && sw.os.ubuntu'
+  build_env:
+    vars:
+      8: 'CC=gcc-7 CXX=g++-7'
+      11: 'CC=gcc-7 CXX=g++-7'
+      12: 'CC=gcc-7 CXX=g++-7'
+      13: 'CC=gcc-7 CXX=g++-7'
+      next: 'CC=gcc-7 CXX=g++-7'
+  extra_getsource_options:
+    8: '--openssl-version=1.1.1d'
+    11: '--openssl-version=1.1.1d'
+    12: '--openssl-version=1.1.1d'
+    13: '--openssl-version=1.1.1d'
+    next: '--openssl-version=1.1.1d'
+  extra_configure_options:
+    8: '--with-openssl=fetched --enable-jitserver=yes'
+    11: '--with-openssl=fetched --enable-jitserver=yes'
+    12: '--with-openssl=fetched --enable-jitserver=yes'
+    13: '--with-openssl=fetched --enable-jitserver=yes'
+    next: '--with-openssl=fetched --enable-jitserver=yes'
+  excluded_tests:
+    11:
+      ? special.system
+    13:
+      ? special.system
+  test_flags:
+    8: 'JITAAS'
+    11: 'JITAAS'
+    12: 'JITAAS'
+    13: 'JITAAS'
+    next: 'JITAAS'
+#========================================#
 # AIX PPC 64bits Compressed Pointers
 #========================================#
 ppc64_aix:
@@ -466,6 +576,63 @@ x86-64_linux_cm:
       <<: *defaultTests
       ? special.system
 #========================================#
+# Linux x86 64bits Compressed Pointers /w JITSERVER
+#========================================#
+x86-64_linux_jit:
+  boot_jdk:
+    8: '/usr/lib/jvm/jdk-7'
+    11: '/usr/lib/jvm/jdk-11'
+    12: '/usr/lib/jvm/jdk-11'
+    13: '/usr/lib/jvm/adoptojdk-java-12'
+    next: '/usr/lib/jvm/adoptojdk-java-13'
+  release:
+    8: 'linux-x86_64-normal-server-release'
+    11: 'linux-x86_64-normal-server-release'
+    12: 'linux-x86_64-server-release'
+    13: 'linux-x86_64-server-release'
+    next: 'linux-x86_64-server-release'
+  freemarker: '/home/jenkins/freemarker.jar'
+  openjdk_reference_repo: '/home/jenkins/openjdk_cache'
+  node_labels:
+    build:
+      8: 'ci.role.build && hw.arch.x86 && sw.os.cent.6'
+      11: 'ci.role.build && hw.arch.x86 && sw.os.cent.6'
+      12: 'ci.role.build && hw.arch.x86 && sw.os.cent.6'
+      13: 'ci.role.build && hw.arch.x86 && sw.os.cent.6'
+      next: 'ci.role.build && hw.arch.x86 && sw.os.cent.6'
+  extra_getsource_options:
+    8: '--openssl-version=1.1.1d'
+    11: '--openssl-version=1.1.1d'
+    12: '--openssl-version=1.1.1d'
+    13: '--openssl-version=1.1.1d'
+    next: '--openssl-version=1.1.1d'
+  extra_configure_options:
+    8: '--enable-cuda --with-cuda=/usr/local/cuda-9.0 --with-openssl=fetched --enable-jitserver=yes'
+    11: '--enable-cuda --with-cuda=/usr/local/cuda-9.0 --with-openssl=fetched --enable-jitserver=yes'
+    12: '--enable-cuda --with-cuda=/usr/local/cuda-9.0 --with-openssl=fetched --enable-jitserver=yes'
+    13: '--enable-cuda --with-cuda=/usr/local/cuda-9.0 --with-openssl=fetched --enable-jitserver=yes'
+    next: '--enable-cuda --with-cuda=/usr/local/cuda-9.0 --with-openssl=fetched --enable-jitserver=yes'
+  build_env:
+    cmd:
+      8: 'source /opt/rh/devtoolset-7/enable'
+      11: 'source /opt/rh/devtoolset-7/enable'
+      12: 'source /opt/rh/devtoolset-7/enable'
+      13: 'source /opt/rh/devtoolset-7/enable'
+      next: 'source /opt/rh/devtoolset-7/enable'
+  excluded_tests:
+    11:
+      ? special.system
+    12:
+      ? special.system
+    13:
+      ? special.system
+  test_flags:
+    8: 'JITAAS'
+    11: 'JITAAS'
+    12: 'JITAAS'
+    13: 'JITAAS'
+    next: 'JITAAS'
+#========================================#
 # Linux x86 64bits Large Heap
 #========================================#
 x86-64_linux_xl: