From 4523e666bcc0247474f2ecb5a1a3264e3faa31c6 Mon Sep 17 00:00:00 2001 From: Marouane El Hallaoui Date: Mon, 29 Sep 2025 13:39:32 +0100 Subject: [PATCH] simplify one-off building process --- .ci/check_devkit_versions.py | 2 +- .ci/ci.jsonnet | 563 +++++++++++++++++++++++++++++++++++ .ci/common.libsonnet | 279 +++++++++++++++++ ci.jsonnet | 408 ------------------------- 4 files changed, 843 insertions(+), 409 deletions(-) create mode 100644 .ci/ci.jsonnet create mode 100644 .ci/common.libsonnet delete mode 100644 ci.jsonnet diff --git a/.ci/check_devkit_versions.py b/.ci/check_devkit_versions.py index 5a0d915c07f..3a75bafd8d7 100644 --- a/.ci/check_devkit_versions.py +++ b/.ci/check_devkit_versions.py @@ -36,7 +36,7 @@ from os.path import dirname, join repo_dir = dirname(dirname(__file__)) -ci_jsonnet_path = join(repo_dir, 'ci.jsonnet') +ci_jsonnet_path = join(repo_dir, '.ci/ci.jsonnet') jib_profiles_path = join(repo_dir, 'make', 'conf', 'jib-profiles.js') def load_jib_devkits(): diff --git a/.ci/ci.jsonnet b/.ci/ci.jsonnet new file mode 100644 index 00000000000..a29ecb869df --- /dev/null +++ b/.ci/ci.jsonnet @@ -0,0 +1,563 @@ +# To process this file using Jsonnet commandline interpreter +# Use the following command: jsonnet .ci/ci.jsonnet + +# https://github.com/graalvm/labs-openjdk/blob/master/doc/testing.md +local run_test_spec = 'test/hotspot/jtreg/compiler/jvmci test/jdk/tools/jlink/plugins/SaveJlinkArgfilesPluginTest.java'; + + +local common = import 'common.libsonnet'; + +{ + overlay: '3bc18600dc70191a267a14f17e817fd071a2d277', + specVersion: '3', + + local prebuild_artifact_url = 'jdk_tlda.archive.artifacts', + Prebuild(major_java_version, release, defs):: common.LinuxAMD64(defs, major_java_version) + common.AMD64 + common.DefaultBootJDK + common.Packages + common.mxDependencies { + name: 'prebuild-labsjdk-%s-ee-%s-linux-amd64' % [major_java_version, release.name], + targets: ['dependency'], + components+: ['build-labsjdk-snapshots-%s' % release.name], + logs: [ + '*.log', + '*/build/*/configure-support/sign-dummy/sign-dummy.signlog', + ], + deploysArtifacts: true, + python_version: '3', + diskspace_required: '10G', + setup: [ + ['set-export', 'BUILD_MAIN', '$PWD'], + ['set-export', 'JIB_DATA_DIR', '${BUILD_MAIN}/../jib'], + ['set-export', 'JIB_SERVER', defs.jib_server], + ['set-export', 'JIB_SERVER_MIRRORS', defs.jib_server_mirrors], + ], + timelimit: '45:00', + notify_emails: [defs.build_failure_notify], + publishArtifacts: [ + { + name: 'jdk-%s-%s-%s_tlda' % [release.name, major_java_version, release.version], + dir: '.', + patterns: [prebuild_artifact_url], + }, + ], + + run+: + # Clone JDK sources + common.clone_labsjdk_builder(defs) + + common.clone(defs, 'labsjdk-ee', release, 'labsjdk-ee', is_closed=true) + + common.clone(defs, 'labsjdk-ce', release, 'labsjdk-ee/open') + + [ + ['set-export', 'JAVA_VERSION', ['python', '-c', 'import sys; sys.path.insert(0, "${LABSJDK_BUILDER_DIR}"); import util; print(util.get_java_version(util.get_version_numbers_file("./labsjdk-ee/open")))']], + ['set-export', 'JAVA_VERSION_BUILD', ['python', '-c', 'import sys; sys.path.insert(0, "${LABSJDK_BUILDER_DIR}"); import util; print(util.get_jdk_build_num("./labsjdk-ee/open", "${JAVA_VERSION}",show_output=False))']], + [ 'python3', '-u', '${LABSJDK_BUILDER_DIR}/prebuild_labsjdk.py', + '--uploader=${ARTIFACT_UPLOADER_SCRIPT}', + '--notify-email', defs.upload_failure_notify, + '--artifact-repo-key=${ARTIFACT_REPO_KEY_LOCATION}', + '--patches=patches.tar.gz', + '--fingerprint=' + 'prebuild.fingerprint', + '--graalvm-version=%s' % release.name, + '--check-exists', + release.version, + 'labsjdk-ee', + 'jdk_tlda', + ], + ['cat', prebuild_artifact_url], + ], + }, + + + # Creates a builder object. + # + # number major_java_version: major Java version (e.g. 25) + # string edition: "ce" or "ee" + # object platform: details of a platform including the packages, docker image or devkit supplying the dependencies + Build(major_java_version, edition, release, platform, static_libs, llvm, defs):: platform + common.mxDependencies + common.Packages { + local env = platform.environment, + local os = platform.os, + local arch = platform.arch, + local exe_suffix = if os == 'windows' then '.exe' else '', + local sep = if os == 'windows' then '\\' else '/', + + local path(p) = if os == 'windows' then std.strReplace(p, '/', '\\') else p, + local exe(p) = if os == 'windows' then p + '.exe' else p, + local jdk_src_dir = if edition == 'ee' then 'jdk_tlda' else 'labsjdk-' + edition, + local type = if static_libs then 'static-libs' else if llvm then 'llvm' else 'snapshot', + local is_windows(os) = os == 'windows', + + name: '%s-labsjdk-%s-%s-%s-%s-%s' % [type, major_java_version, edition, release.name, os, arch], + targets: ['ondemand'], + logs: [ + '*.log', + '*/build/*/configure-support/sign-dummy/sign-dummy.signlog', + ], + diskspace_required: '10G', + deploysArtifacts: true, + python_version: '3', + components+: ['build-labsjdk-snapshots-%s' % release.name], + environment+: { + ARTIFACT_REPO_URL: defs.artifacts_service_url, + MX_PYTHON_VERSION: '3', + MX_PYTHON: 'python3', + }, + setup: common.clone_labsjdk_builder(defs, is_windows(os)) + [ + ['set-export', 'BUILD_MAIN', '$PWD'], + ['set-export', 'JIB_DATA_DIR', path('${BUILD_MAIN}/../jib')], + ['set-export', 'JIB_SERVER', defs.jib_server], + ['set-export', 'JIB_SERVER_MIRRORS', defs.jib_server_mirrors], + ] + (if edition == 'ce' then + common.clone(defs, 'labsjdk-ce', release, 'labsjdk-ce', is_windows(os)) + [ + ['python3', '${LABSJDK_BUILDER_DIR}/snapshot_id.py', '-o', 'labsjdk-ce/snapshot_id.txt', 'labsjdk-ce'], + ['set-export', 'JAVA_VERSION', ['python', '-c', 'import sys; sys.path.insert(0, "${LABSJDK_BUILDER_DIR}"); import util; print(util.get_java_version(util.get_version_numbers_file("./labsjdk-ce")))']], + ['set-export', 'JAVA_VERSION_BUILD', ['python', '-c', 'import sys; sys.path.insert(0, "${LABSJDK_BUILDER_DIR}"); import util; print(util.get_jdk_build_num("./labsjdk-ce", "${JAVA_VERSION}",show_output=False))']], + ['set-export', 'LLVM_BUNDLE_EDITION', ''], + ['set-export', 'LLVM_BUNDLE_DIRECTORY_PREFIX', '-community'], + + ] + else + common.clone(defs, 'labsjdk-ce', release, 'labsjdk-ee-open', is_windows(os)) + [ + ['set-export', 'JAVA_VERSION', ['python', '-c', 'import sys; sys.path.insert(0, "${LABSJDK_BUILDER_DIR}"); import util; print(util.get_java_version(util.get_version_numbers_file("./labsjdk-ee-open")))']], + ['set-export', 'JAVA_VERSION_BUILD', ['python', '-c', 'import sys; sys.path.insert(0, "${LABSJDK_BUILDER_DIR}"); import util; print(util.get_jdk_build_num("./labsjdk-ee-open", "${JAVA_VERSION}",show_output=False))']], + ['set-export', 'LLVM_BUNDLE_EDITION', '-svmee'], + ['set-export', 'LLVM_BUNDLE_DIRECTORY_PREFIX', ''], + + # Download TLDA source archive from prebuild step for EE labsjdk + ['set-export', 'TLDA_URL', ['cat', prebuild_artifact_url]], + ['python3', '-c', 'from urllib.request import urlretrieve;urlretrieve( "$TLDA_URL", "jdk_tlda.tar.gz")'], + + # Extract the TLDA sources. The -m options avoids issues with the + # clock skew between the machine producing the archive and the + # machine running this builder. This prevents issues when running + # make such as: + # + # Error: The configuration is not up to date for 'linux-aarch64'. + # + ['tar', 'xzfm', 'jdk_tlda.tar.gz'], + + ]) + + (if llvm && os == 'darwin' && arch == 'aarch64' then [ + ['curl', '-L', defs.graalvm_llvm_standalone_macos_aarch64_url, '|', 'tar', 'xz'], + ['set-export', 'TOOLCHAIN_GRAALVM', '$PWD/llvm${LLVM_BUNDLE_DIRECTORY_PREFIX}-24.1.0-macos-aarch64'], + ['set-export', 'LLVM_TOOLCHAIN_LLI', '${TOOLCHAIN_GRAALVM}/bin/lli'], + ] else []) + + (if llvm && os == 'linux' && arch == 'amd64' then [ + + ['curl', '-L', defs.graalvm_base_linux_url, '|', 'tar', 'xz'], + ['set-export', 'TOOLCHAIN_GRAALVM', '$PWD/graalvm-ee-java11-22.1.0-dev'], + [platform.java_home('$TOOLCHAIN_GRAALVM') + '/bin/gu', 'install', '-u', defs.graalvm_llvm_toolchain_installable_url], + ['set-export', 'LLVM_TOOLCHAIN_LLI', platform.java_home('$TOOLCHAIN_GRAALVM') + '/bin/lli'], + + ] else []), + + timelimit: if llvm && os == 'darwin' then '4:30:00' else '3:30:00', + notify_emails: [defs.build_failure_notify], + + local required_static_libs_platforms = common.get(common.get(platform, 'required_static_libs'), edition, []), + local release_qualifier = release.name, + + local publishStaticLibsArtifacts = if static_libs then [{ + name: 'static-libs-' + edition + '-' + major_java_version + release_qualifier + '-' + os + '-' + arch, + dir: '.', + patterns: ['__ci_artifact_bookmarks'], + }] else [], + + local requireStaticLibsArtifacts = [{ + name: 'static-libs-' + edition + '-' + major_java_version + release_qualifier + '-' + name, + dir: '.', + } for name in required_static_libs_platforms], + + local requireTLDAArtifacts = if edition == 'ee' then [ + { + name: 'jdk-%s-%s-%s_tlda' % [release.name, major_java_version, release.version], + dir: '.', + }, + ] else [], + + publishArtifacts: publishStaticLibsArtifacts, + requireArtifacts: requireStaticLibsArtifacts + requireTLDAArtifacts, + local build(level, check_exists, major_java_version, build_number) = [ 'python3', '-u', '${LABSJDK_BUILDER_DIR}/build_labsjdk.py', + '--bundles=' + if static_libs + then 'static-libs' + else if llvm then 'product,test' + else 'product,static-libs,test', + '--log=' + if os == 'windows' then 'info' else 'warn', + '--java-major-version=' + major_java_version, + '--java-build-number=' + build_number, + '--jvmci-version=' + release.version, + '--boot-jdk=${BOOT_JDK}', + '--patches=' + jdk_src_dir + '/patches.tar.gz', + '--clean-after-build', + '--jdk-debug-level=' + level, + '--java-home-link-target=' + level + '-java-home', + '--fingerprint=' + level + '.fingerprint', + '--uploader=${ARTIFACT_UPLOADER_SCRIPT}', + '--graalvm-version=%s' % release.name, + '--notify-email', defs.upload_failure_notify, + '--artifact-repo-key=${ARTIFACT_REPO_KEY_LOCATION}', + '--snapshot-id=' + path(jdk_src_dir + '/snapshot_id.txt'), + jdk_src_dir, + ] + + (if edition == 'ee' then [ + '--jdk-build-num=' + path(jdk_src_dir + '/build_num.txt'), + ] else []) + + (if llvm then [ + '--sulong-toolchain=$LLVM_TOOLCHAIN_LLI', + '--no-tests', # do not need test bundles for llvm-labsjdk + ] else [ + '--ci-artifact-bookmarks=__ci_artifact_bookmarks', + '--test=test/hotspot/jtreg/compiler/jvmci', + ]) + + (if check_exists then ['--check-exists'] else []) + + (if platform.arch == 'riscv64' then [ + '--target-platform=linux-riscv64', + '--sysroot=$RISCV_HOME/fedora28_riscv_root', + ] else []), + + run+: [ + build('release', true, '${JAVA_VERSION}', '${JAVA_VERSION_BUILD}'), + build('fastdebug', false, '${JAVA_VERSION}', '${JAVA_VERSION_BUILD}'), + ], + }, + + local produce_builds(major_java_version, releases, static_libs, llvm, defs) = [ + self.Build(major_java_version, edition, release, platform, static_libs, llvm, defs) + for release in releases + for platform in ( + if static_libs then static_libs_platforms(major_java_version, release, defs) + else if llvm then llvm_platforms(defs, major_java_version, release) + else jdk_platforms(major_java_version, release, defs) + ) + for edition in platform.editions + ], + + # Platform configs for builders that only produce static libs. + local static_libs_platforms(major_java_version, release, defs) = [ + # linux-amd64-musl static libs are only built for CE + common.LinuxDockerAMD64Musl(defs) + common.MuslBootJDK + { editions: ['ce', 'ee'] }, + ], + + # Platform configs for builders that produce llvm bitcode libs. + local llvm_platforms(defs, major_java_version, release) = [ + common.LinuxAMD64(defs, major_java_version) + common.AMD64 + common.DefaultBootJDK + { + editions: ['ce', 'ee'], + }, + common.DarwinAArch64 + common.DefaultBootJDK + { + editions: ['ce', 'ee'], + }, + ], + + # Platform configs for JDK image builders. + local jdk_platforms(major_java_version, release, defs) = [ + # linux-amd64 has CE and EE builds and the CE builds include the linux-amd64-musl static libs + common.LinuxAMD64(defs, major_java_version) + common.AMD64 + common.DefaultBootJDK + + { editions: ['ce', 'ee'] } + + { required_static_libs: + {ce: ['linux-amd64_musl'], + ee: ['linux-amd64_musl'], + }, + }, + + common.LinuxAArch64(defs, major_java_version) + common.AArch64 + common.JTReg + common.DefaultBootJDK + + { editions: ['ce', 'ee'] }, + + # darwin-aarch64 has CE and EE builds on 11 and 16+ + common.DarwinAArch64 + { editions: ['ce', 'ee'] }, + + common.WindowsCygwin + common.AMD64 + common.DefaultBootJDK + + { editions: ['ce', 'ee'] }, + + ] + static_libs_platforms(major_java_version, release, defs), # For building musl boot JDKs + + SourceBuild(java_version, release, defs):: common.LinuxAMD64(defs, java_version) + common.AMD64 + common.OCI + common.DefaultBootJDK + common.Packages + common.mxDependencies + { + local jvmci_build_number = release.version, + local repo_name = 'labsjdk-ee', + local jdk_conf_name = 'linux-x86_64-server-release', + local src_bundle_suffixes = ['', '-security'], + + copy_source_bundle(suffix=''):: ['cp', '${JDK_SOURCE}/' + repo_name + '/build/' + jdk_conf_name + '/source-bundles/jdk-${JAVA_VERSION}+${JAVA_VERSION_BUILD}_src' + suffix + '.zip', '../results/'], + unzip_source_bundle(suffix=''):: ['unzip', '-q', '../results/jdk-${JAVA_VERSION}+${JAVA_VERSION_BUILD}_src' + suffix + '.zip'], + + docker: { + image: defs.linux_docker_image_amd64, + }, + targets+: ['ondemand'], + python_version: '3', + deploysArtifacts: true, + components+: ['build-labsjdk-snapshots-%s' % release.name], + name: 'source-labsjdk-%s-ee-%s-%s-%s' % [java_version, release.name, self.os, self.arch], + setup+: common.clone_labsjdk_builder(defs) + [ + ['set-export', 'BUILD_MAIN', '.'], + ['mkdir', '../results'], + ['mkdir', '../jvmci'], + ['set-export', 'JDK_SOURCE', ['realpath', '..']], + ], + run+: + common.clone(defs, repo_name, release, '$JDK_SOURCE/' + repo_name, is_closed=true) + + [ + ['git', '-C', '$JDK_SOURCE/' + repo_name, 'clone', defs.labsjdk_ce_url, 'open'], + ['git', '-C', '$JDK_SOURCE/' + repo_name + '/open', 'checkout', '${MAIN_REVISION}'], + ['set-export', 'SNAPSHOT_ID', ['python3', '${LABSJDK_BUILDER_DIR}/snapshot_id.py', '$JDK_SOURCE/' + repo_name + '/open', '$JDK_SOURCE/' + repo_name]], + ['set-export', 'JAVA_VERSION', ['python', '-c', 'import sys; sys.path.insert(0, "${LABSJDK_BUILDER_DIR}"); import util ;print(util.get_java_version(util.get_version_numbers_file("$JDK_SOURCE/labsjdk-ee/open")))']], + ['set-export', 'JAVA_VERSION_BUILD', ['python', '-c', 'import sys; sys.path.insert(0, "${LABSJDK_BUILDER_DIR}"); import util; print(util.get_jdk_build_num("$JDK_SOURCE/labsjdk-ee/open", "${JAVA_VERSION}",show_output=False))']], + ['cd', '$JDK_SOURCE/' + repo_name], + [ 'sh', 'configure', + '--with-devkit=$DEVKIT_ROOT', + '--with-boot-jdk=$BOOT_JDK', + '--with-jvm-features=compiler1,compiler2,jvmci', + '--enable-jvm-feature-shenandoahgc', + '--with-version-build=${JAVA_VERSION_BUILD}', + '--with-version-pre=', + '--with-version-opt=LTS-jvmci-' + release.name + '-' + release.version, + ], + ['make', 'source'], + ] + + [self.copy_source_bundle(suffix) for suffix in src_bundle_suffixes] + + [ + ['zip', '-ur', '../results/jdk-${JAVA_VERSION}+${JAVA_VERSION_BUILD}_src.zip', '$BUILD_MAIN/closed/make/conf/*'], + ['tar', 'czf', '../' + 'labsjdk-ee-${JAVA_VERSION}+${JAVA_VERSION_BUILD}-jvmci-%s-%s-${SNAPSHOT_ID}-src.tar.gz' % [release.name, release.version], '../results'], + ['cd', '$JDK_SOURCE/jvmci'], + ] + + [self.unzip_source_bundle(suffix) for suffix in src_bundle_suffixes] + + [ + [ 'sh', 'configure', + '--with-version-pre=', + '--with-version-opt=jvmci-' + release.version, + '--with-devkit=$DEVKIT_ROOT', + '--with-boot-jdk=$BOOT_JDK', + '--with-version-build=${JAVA_VERSION_BUILD}', + '--with-zlib=system', + '--enable-dtrace=no', + '--disable-warnings-as-errors', + ], + ['make', 'images', 'static-libs-bundles'], + [ + 'artifact_uploader.py', + '../' + 'labsjdk-ee-${JAVA_VERSION}+${JAVA_VERSION_BUILD}-jvmci-%s-%s-${SNAPSHOT_ID}-src.tar.gz' % [release.name, release.version], + 'labsjdk/' + 'labsjdk-ee-${JAVA_VERSION}+${JAVA_VERSION_BUILD}-jvmci-%s-%s-${SNAPSHOT_ID}-src.tar.gz' % [release.name, release.version], + 'labsjdk', + '--version', '${JAVA_VERSION}+${JAVA_VERSION_BUILD}-jvmci-%s-%s-${SNAPSHOT_ID}' % [release.name, release.version], + '--jdk', java_version + '', + '--revision', ['git', '-C', '$JDK_SOURCE/' + repo_name + '/open', 'rev-parse', 'HEAD'], + '--edition', 'ee', + '--artifact-type', 'source', + '--lifecycle', 'snapshot', + '--lifecycle', 'snapshot', + ], + ], + }, + + local setupJDKSources(conf) = { + run+: [ + # To reduce load, the CI system does not fetch all tags so it must + # be done explicitly as `build_labsjdk.py` relies on it. + ['git', 'fetch', '-q', '--tags'], + ] + (if conf.os == 'windows' then [ + # Need to fix line endings on Windows to satisfy cygwin + # https://stackoverflow.com/a/26408129 + ['set-export', 'JDK_SRC_DIR', '${PWD}\\..\\jdk'], + ['git', 'clone', '--quiet', '--config', 'core.autocrlf=input', '-c', 'gc.auto=0', '.', '${JDK_SRC_DIR}'], + ] else [ + ['set-export', 'JDK_SRC_DIR', '${PWD}'], + ]) + [ + ['set-export', 'JDK_SUITE_DIR', '${JDK_SRC_DIR}'], + ], + }, + + CreateGraalIntegrationPR(release, defs):: common.Linux + common.AMD64 + common.OCI { + name: 'graal-integration-automation-release-%s' % release.name, + targets: ['ondemand'], + notify_emails: [defs.build_failure_notify], + packages+: { + 'pip:requests': '2.25.1', + 'pip:python-dateutil': '2.8.2', + }, + capabilities+: ['e4_36_256'], + docker: { + image: defs.linux_docker_image_amd64, + }, + python_version: '3', + run: common.clone_labsjdk_builder(defs, false) + [ + ['set-export', 'BUILD_MAIN', '$PWD'], + ['cp', '${LABSJDK_BUILDER_DIR}/update_ci_imports.py', '.'], + ['cp', '${LABSJDK_BUILDER_DIR}/util.py', '.'], + ['cp', '${LABSJDK_BUILDER_DIR}/snapshot_id.py', '.'], + ['cp', '${LABSJDK_BUILDER_DIR}/release_labsjdks.py', '.'], + ['git', 'config', '--global', 'user.email', defs.ol_automation_user], + ['git', 'config', '--global', 'user.name', 'ol-automation_ww'], + ['git', 'clone', '--config', 'core.autocrlf=input', '--branch', 'master', '--quiet', defs.graal_url, '$BUILD_MAIN/graal'], + ['git', 'clone', '--config', 'core.autocrlf=input', '--branch', 'master', '--quiet', defs.graal_enterprise_url, '$BUILD_MAIN/graal-enterprise'], + ['git', 'clone', '--config', 'core.autocrlf=input', '--branch', 'master', '--quiet', defs.graalos_url, '$BUILD_MAIN/graalos'], + ['git', 'clone', '--config', 'core.autocrlf=input', '--quiet', defs.buildbot_ci_url, '$BUILD_MAIN/buildbot-ci'], + + [ 'python3', '-u', '${LABSJDK_BUILDER_DIR}/graal_automation.py', + '--buildbot-ci-path', '$BUILD_MAIN/buildbot-ci/pr_creator.py', + '--graal-path', '$BUILD_MAIN/graal', + '--graal-enterprise-path', '$BUILD_MAIN/graal-enterprise', + '--graalos-path', '$BUILD_MAIN/graalos', + '--target-branch', 'master', + '--candidate-jvmci-release', '%s' % release.candidate_jvmci_release, + '--reviewers', defs.graal_prs_default_reviewers + ',' + '${BLAME_LIST}', + ], + ], + }, + + GraalDeployAutomationJob(release, defs):: common.Linux + common.AMD64 + common.OCI { + name: 'labsjdk-deploy-automation-release-%s' % release.name, + targets: ['ondemand'], + notify_emails: [defs.build_failure_notify], + packages+: { + 'pip:requests': '2.25.1', + 'pip:python-dateutil': '2.8.2', + }, + docker: { + image: defs.linux_docker_image_amd64, + }, + capabilities+: ['e4_36_256'], + python_version: '3', + run: common.clone_labsjdk_builder(defs, false) + [ + ['set-export', 'BUILD_MAIN', '$PWD'], + ['set-export', 'HTTP_HTTPS_PROXY', defs.proxy_url], + ['cp', '${LABSJDK_BUILDER_DIR}/release_labsjdks.py', '.'], + ['cp', '${LABSJDK_BUILDER_DIR}/util.py', '.'], + ['cp', '${LABSJDK_BUILDER_DIR}/snapshot_id.py', '.'], + + ['python3', '-u', '${LABSJDK_BUILDER_DIR}/deploy_automation.py', + '--graal-workdir', '$BUILD_MAIN', + '--candidate-jvmci-release', '%s' % release.candidate_jvmci_release, + '--graal-integration-branch', 'master', + '--allow-missing', 'patches', + '--upload-ce-artifact', '%s' % release.github_upload_labsjdk_ce, + '--pass-snapshots-file', 'true', + ], + ], + }, + + BuildValidation(defs, conf, is_musl_build, graalvm_version):: conf + setupJDKSources(conf) + (if is_musl_build then common.MuslBootJDK else common.DefaultBootJDK) + { + name: 'gate-build-jdk' + conf.name, + timelimit: '3:00:00', # Windows is the long pole + diskspace_required: '10G', + logs: ['*.log'], + targets: ['gate'], + deploysArtifacts: true, + + local build_labsjdk(jdk_debug_level, java_home_env_var) = [ + ['set-export', java_home_env_var, conf.path('${PWD}/../%s-java-home' % jdk_debug_level)], + ['python3', '-u', conf.path('${LABSJDK_BUILDER_DIR}/build_labsjdk.py'), + '--boot-jdk=${BOOT_JDK}', + '--clean-after-build', + '--uploader=disabled', + '--jdk-debug-level=' + jdk_debug_level, + '--jvmci-version=b01', + '--graalvm-version=%s' % graalvm_version, + '--test=' + run_test_spec, + '--java-home-link-target=${%s}' % java_home_env_var, + ] + (if is_musl_build then ['--bundles=static-libs'] else []) + + ['${JDK_SRC_DIR}'], + ], + + run+: (if !is_musl_build then [ + # Checks that each devkit mentioned in this file corresponds to a devkit in make/conf/jib-profiles.js + ['python3', '-u', conf.path('${PWD}/.ci/check_devkit_versions.py')], + ] else []) + [ + ['set-export', 'LABSJDK_BUILDER_DIR', conf.path('${PWD}/../labsjdk-builder')], + ['git', 'clone', '--quiet', '--config', 'core.autocrlf=input', defs.labsjdk_builder_url, '${LABSJDK_BUILDER_DIR}'], + ['git', '-C', '${LABSJDK_BUILDER_DIR}', 'checkout', common.labsjdk_builder_version], + + # This restricts cygwin to be on the PATH only while using jib. + # It must not be on the PATH when building Graal. + ['set-export', 'OLD_PATH', '${PATH}'], + ['set-export', 'PATH', '${JIB_PATH}'], + ['set-export', 'JIB_SERVER', defs.jib_server], + ['set-export', 'JIB_SERVER_MIRRORS', defs.jib_server_mirrors], + ['set-export', 'JIB_DATA_DIR', conf.path('${PWD}/../jib')], + ] + + build_labsjdk('release', 'JAVA_HOME') + + build_labsjdk('fastdebug', 'JAVA_HOME_FASTDEBUG'), + }, + + local build_confs(defs) = [ + common.LinuxAMD64Graal(defs, true), + common.LinuxAArch64Graal(defs, true), + common.DarwinAArch64, + common.Windows + common.AMD64, + ], + + local amd64_musl_confs(defs) = [ + common.LinuxDockerAMD64Musl(defs), + ], + + DefineBuilds(defs):: [self.BuildValidation(defs, conf, is_musl_build=false, graalvm_version=25.1) for conf in build_confs(defs)] + + [self.BuildValidation(defs, conf, is_musl_build=true, graalvm_version=25.1) for conf in amd64_musl_confs(defs)], + + CreateBuilds(major_java_version, releases, defs):: + [self.Prebuild(major_java_version, release, defs) for release in releases] + + [self.SourceBuild(major_java_version, release, defs) for release in releases] + + [self.CreateGraalIntegrationPR(release, defs) for release in releases] + + [self.GraalDeployAutomationJob(release, defs) for release in releases] + + + produce_builds(major_java_version, releases, false, false, defs) + + produce_builds(major_java_version, releases, true, false, defs) + + produce_builds(major_java_version, releases, false, true, defs), + + # These are configuration values that must be supplied externally (e.g. in an overlay) + local defs = { + # Linux Docker image based on Alpine 3.15 (based on musl-1.2.2). + linux_docker_image_amd64_musl: 'artifacts.acme.com/docker/images/buildslave:gcc-10.3.1-musl-1.2.2', + + # Oracle Linux 8 Docker image for AMD64 architecture. + linux_docker_image_amd64: 'artifacts.acme.com/docker/images/buildslave:ol8', + + # Primary JIB (JDK Integrated Builder) server URL. + jib_server: 'https://build.dependencies.acme.com/artifactory', + + # Mirrors JIB (Java Image Builder) server URL. + jib_server_mirrors: 'https://build.dependencies2.acme.com/artifactory https://build.dependencies3.acme.com/artifactory', + + # Internal artifacts service used to store the labsjdk binaries. + artifacts_service_url: 'https://build.dependencies3.acme.com/artifactory', + + # GraalVM base image for the Linux AMD64 platform. + graalvm_base_linux_url: 'https://acme.com/graalvm-bundle-release-generic-local/graalvm-java11-linux-amd64-22.1.0.tar.gz', + + # GraalVM LLVM toolchain installable for the Linux AMD64 platform. + graalvm_llvm_toolchain_installable_url: 'https://acme.com/graalvm-bundle-release-generic-local/content/24.1.0/llvm-native-standalone-svm-java23-linux-amd64-24.1.0-signed.tar.gz', + + # GraalVM LLVM standalone toolchain tarball for macOS AArch64 (Apple Silicon). + graalvm_llvm_standalone_macos_aarch64_url: 'https://acme.com/graalvm-bundle-release-generic-local/content/24.1.0/llvm-native-standalone-svm-java23-darwin-aarch64-24.1.0-signed.tar.gz', + + # Recipient(s) to notify in case of upload failures. + upload_failure_notify: 'reviewer1@oracle.com,reviewer2@oracle.com', + + # List of recipient to notify in case of build failures. + build_failure_notify: 'reviewer1@oracle.com,reviewer2@oracle.com', + + # List of default reviewers for automated Graal pull requests. + graal_prs_default_reviewers: 'reviewer1@oracle.com,reviewer2@oracle.com', + + # Graal Enterprise repository. + graal_enterprise_url: 'ssh://git@bitbucket.acme.com:8999/graal/graal-enterprise.git.', + + # Graal repository + graal_url: 'ssh://git@bitbucket.acme.com:8999/graal/graal.git.', + + # GraalOS repository. + graalos_url: 'ssh://git@bitbucket.acme.com:8999/graal/graalos.git.', + + # Buildbot CI repository. + buildbot_ci_url: 'ssh://git@bitbucket.acme.com:8999/graal/buildbot-ci.git.', + + # labsjdk-builder repository. + labsjdk_builder_url: 'ssh://git@bitbucket.acme.com:8999/graal/labsjdk-builder.git.', + + # labsjdk community edition repository. + labsjdk_ce_url: 'ssh://git@bitbucket.acme.com:8999/graal/labsjdk-ce.git', + + # ol-automation user mail. + ol_automation_user: 'ol@oracle.com', + + # Base URL for Bitbucket repositories + bitbucket_base_url: 'ssh://git@bitbucket.acme.com:8999/graal/', + + # Proxy server URL. + proxy_url: 'http://proxy.acme.com:8090', + }, + builds: self.DefineBuilds(defs) + self.CreateBuilds(25, common.jvmci_releases, defs), +} diff --git a/.ci/common.libsonnet b/.ci/common.libsonnet new file mode 100644 index 00000000000..80ad4ae572b --- /dev/null +++ b/.ci/common.libsonnet @@ -0,0 +1,279 @@ +# One-off snapshots metadata +local one_off_release = { + name: '25.1', + version: 'b10', + candidate_jvmci_release: '25.0.1+8-jvmci-25.1-b10', + github_upload_labsjdk_ce: 'true', + java_versions: [25], + musl: [25], + llvm: [25], +}; + + +{ + + labsjdk_builder_version:: '243066362288ec0c8557bf96140c2578630448cd', + +# This function is used to clone closed and open JDK repository +# Always checkout branch jdk25 for labsjdk-ee repository +# Always checkout the revision defined in MAIN_REVISION environement variable for labsjdk-ce repository + clone(defs, repo, release, dst_dir, is_windows=false, is_closed=false):: [ + ['git', 'clone', '--config', 'core.autocrlf=input', '--quiet', defs.bitbucket_base_url + repo + '.git', dst_dir], +] + (if is_closed then [ + ['git', '-C', dst_dir, 'checkout', 'jdk25'], +] else [ + ['git', '-C', dst_dir, 'checkout', '${MAIN_REVISION}'], +]), + +# This function is used to clone labsjdk-builder repository + clone_labsjdk_builder(defs, is_windows=false):: (if !is_windows then [ + ['set-export', 'LABSJDK_BUILDER_DIR', '${PWD}/labsjdk-builder'], +] else [ + ['set-export', 'LABSJDK_BUILDER_DIR', '${PWD}' + '\\' + 'labsjdk-builder'], +]) + [ + ['git', 'clone', '--quiet', '--config', 'core.autocrlf=input', defs.labsjdk_builder_url, '${LABSJDK_BUILDER_DIR}'], + ['git', '-C', '${LABSJDK_BUILDER_DIR}', 'checkout', self.labsjdk_builder_version], +], + + # Returns the value of the `name` field if it exists in `obj` otherwise `default`. + get(obj, name, default=null):: + if obj == null then default else + if std.objectHas(obj, name) then obj[name] else default, + + # Specifies a JVMCI release currently in development and + JVMCIRelease(name, version, java_versions, candidate_jvmci_release, github_upload_labsjdk_ce, has_musl_static_libs, has_llvm_libs):: { + name: name, + version: version, + java_versions: java_versions, + candidate_jvmci_release: candidate_jvmci_release, + github_upload_labsjdk_ce: github_upload_labsjdk_ce, + has_musl_static_libs: has_musl_static_libs, + has_llvm_libs: has_llvm_libs, + }, + jvmci_releases:: [self.JVMCIRelease(one_off_release.name, one_off_release.version, one_off_release.java_versions, one_off_release.candidate_jvmci_release, one_off_release.github_upload_labsjdk_ce, true, false)], + + # Returns true if `str` contains `needle` as a substring. + contains(str, needle):: std.findSubstr(needle, str) != [], + + Packages:: { + packages+: { + 'pip:requests': '2.25.1', + }, + }, + + mxDependencies:: { + python_version: '3', + packages+: { + mx: 'HEAD', + }, + }, + + OSBase:: self.mxDependencies { + path(unixpath):: unixpath, + exe(unixpath):: unixpath, + jdk_home(java_home):: self.path(java_home), + java_home(jdk_home):: self.path(jdk_home), + copydir(src, dst):: ['cp', '-r', src, dst], + environment+: { + JIB_PATH: '${PATH}', + MAKE: 'make', + ZLIB_BUNDLING: 'system', + MX_PYTHON: 'python3.8', + }, + packages+: { + cmake: '==3.22.2', + }, + }, + + OCI:: { + capabilities+: ['oci'], + }, + + Windows:: self.OSBase { + path(unixpath):: std.strReplace(unixpath, '/', '\\'), + exe(unixpath):: self.path(unixpath) + '.exe', + # https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/xcopy + copydir(src, dst):: ['xcopy', self.path(src), self.path(dst), '/e', '/i', '/q'], + + downloads+: { + CYGWIN: { name: 'cygwin', version: '3.0.7', platformspecific: true }, + }, + packages+: { + # devkit_platform_revisions in make/conf/jib-profiles.js + 'devkit:VS2022-17.13.2+1': '==0', + }, + capabilities+: ['windows'], + name+: '-windows-cygwin', + os:: 'windows', + environment+: { + JIB_PATH: '$CYGWIN\\bin;$PATH', + ZLIB_BUNDLING: 'bundled', + # Make `msbuild` and common *nix utilities in MKS_HOME available + PATH: 'C:\\Windows\\Microsoft.NET\\Framework64\\v4.0.30319;$MKS_HOME;$PATH', + # Tell `msbuild` to use the environment as is (i.e., the devkit) + UseEnv: 'true', + }, + }, + + WindowsCygwin:: self.OSBase { + downloads+: { + CYGWIN: { name: 'cygwin', version: '3.0.7', platformspecific: true }, + }, + capabilities+: ['windows'], + name+: '-windows-cygwin', + os:: 'windows', + environment+: { + PATH: '$CYGWIN\\bin;$PATH', + }, + }, + Linux:: self.OSBase { + capabilities+: ['linux', '!e3'], + name+: '-linux', + os:: 'linux', + environment+: { + MAKE: 'make', + }, + }, + + LinuxDockerAMD64Musl(defs):: self.Linux + self.OCI + self.AMD64Musl { + docker: { + image: defs.linux_docker_image_amd64_musl, + alpineDns: true, + }, + # No packages are need for building the musl static libs + packages: {}, + }, + + DefaultBootJDK:: { + downloads+: { + BOOT_JDK: { + name: 'jpg-jdk', + version: '24', + build_id: '36', + release: true, + platformspecific: true, + }, + }, + }, + + MuslBootJDK:: { + downloads+: { + BOOT_JDK: { + name: 'labsjdk', + version: 'ee-24+36-jvmci-b01-linux-amd64_musl', + platformspecific: false, + }, + }, + environment+: { + LD_LIBRARY_PATH: '$BOOT_JDK/lib/server', + }, + }, + + JTReg:: { + downloads+: { + JT_HOME: { + name: 'jtreg', + version: '4.2', + }, + }, + }, + + # devkit_platform_revisions in make/conf/jib-profiles.js + get_devkit_platform_revisions(major_java_version, platform):: { + local devkit_platform_revisions = { + '25': { + linux_x64: 'gcc14.2.0-OL6.4+1.0', + macosx_x64: 'Xcode14.3.1+1.0', + windows_x64: 'VS2022-17.13.2+1.0', + linux_aarch64: 'gcc14.2.0-OL7.6+1.0', + linux_arm: 'gcc8.2.0-Fedora27+1.0', + linux_ppc64le: 'gcc8.2.0-Fedora27+1.0', + linux_s390x: 'gcc8.2.0-Fedora27+1.0', + }, + }, + + local value = devkit_platform_revisions[std.toString(major_java_version)][platform], + local name = std.substr(value, 0, std.length(value) - 2), + local version = std.substr(value, std.length(value) - 1, 1), + packages: { + ['devkit:' + name]: '==' + version, + }, + }, + + LinuxAMD64Graal(defs, for_jdk_build):: self.Linux + self.AMD64 { + docker: { + image: defs.linux_docker_image_amd64, + mount_modules: true, # needed for installing the devtoolset package below + }, + packages+: if for_jdk_build then { + # devkit_platform_revisions in make/conf/jib-profiles.js + 'devkit:gcc14.2.0-OL6.4+1': '==0', + } else { + # When building/testing GraalVM, do not use a devkit as it is known not to + # work well when dynamically linking libstdc++. + devtoolset: '==7', + }, + }, + + LinuxAMD64(defs, major_java_version):: self.Linux + self.get_devkit_platform_revisions(major_java_version, 'linux_x64') + self.OCI { + docker: { + image: defs.linux_docker_image_amd64, + }, + }, + + LinuxAArch64(defs, major_java_version):: self.Linux + self.get_devkit_platform_revisions(major_java_version, 'linux_aarch64') + self.OCI + self.AArch64 { + docker: { + image: defs.linux_docker_image_amd64, + mount_modules: true, + }, + }, + + LinuxAArch64Graal(defs, for_jdk_build):: self.Linux + self.AArch64 { + packages+: if for_jdk_build then { + # devkit_platform_revisions in make/conf/jib-profiles.js + 'devkit:gcc14.2.0-OL7.6+1': '==0', + } else { + # See GR-26071 as well as comment in self.LinuxAMD64 + devtoolset: '==7', + }, + docker: { + image: defs.linux_docker_image_amd64, + mount_modules: true, + }, + }, + + Darwin:: self.OSBase { + jdk_home(java_home):: java_home + '/../..', + java_home(jdk_home):: jdk_home + '/Contents/Home', + packages+: { + # No need to specify a "make" package as Mac OS X has make 3.81 + # available once Xcode has been installed. + }, + os:: 'darwin', + name+: '-darwin', + }, + + + DarwinAArch64:: self.Darwin + self.AArch64 + { + capabilities+: ['darwin_ventura'], + }, + + AMD64:: { + capabilities+: ['amd64'], + name+: '-amd64', + arch:: 'amd64', + }, + + AMD64Musl:: self.AMD64 { + name+: '-musl', + arch+: '_musl', + }, + + AArch64:: { + capabilities+: ['aarch64'], + name+: '-aarch64', + arch:: 'aarch64', + }, + + +} diff --git a/ci.jsonnet b/ci.jsonnet deleted file mode 100644 index 993f753b331..00000000000 --- a/ci.jsonnet +++ /dev/null @@ -1,408 +0,0 @@ -# https://github.com/graalvm/labs-openjdk-21/blob/master/doc/testing.md -local run_test_spec = "test/hotspot/jtreg/compiler/jvmci test/jdk/tools/jlink/plugins/SaveJlinkArgfilesPluginTest.java"; - -local labsjdk_builder_version = "181ae3a6077f9bf0a117cc34562c33e871cd4ede"; - -# Returns true if `str` contains `needle` as a substring. -local contains(str, needle) = std.findSubstr(needle, str) != []; - -{ - overlay: "71bd3aa4622661dda0587fdf9000de4bb1029438", - specVersion: "3", - - mxDependencies:: { - python_version: "3", - packages+: { - mx: "HEAD", - python3: "==3.8.10", - 'pip:pylint': '==2.4.4', - }, - }, - - OSBase:: self.mxDependencies + { - path(unixpath):: unixpath, - exe(unixpath):: unixpath, - jdk_home(java_home):: self.path(java_home), - java_home(jdk_home):: self.path(jdk_home), - copydir(src, dst):: ["cp", "-r", src, dst], - environment+: { - JIB_PATH: "${PATH}", - MAKE : "make", - ZLIB_BUNDLING: "system", - MX_PYTHON: "python3.8" - }, - packages+: { - cmake: "==3.22.2", - }, - }, - - Windows:: self.OSBase + { - path(unixpath):: std.strReplace(unixpath, "/", "\\"), - exe(unixpath):: self.path(unixpath) + ".exe", - # https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/xcopy - copydir(src, dst):: ["xcopy", self.path(src), self.path(dst), "/e", "/i", "/q"], - - downloads+: { - CYGWIN: {name: "cygwin", version: "3.0.7", platformspecific: true}, - }, - packages+: { - # devkit_platform_revisions in make/conf/jib-profiles.js - "devkit:VS2022-17.13.2+1" : "==0" - }, - capabilities+: ["windows"], - name+: "-windows-cygwin", - os:: "windows", - environment+: { - JIB_PATH: "$CYGWIN\\bin;$PATH", - ZLIB_BUNDLING: "bundled" - }, - }, - Linux:: self.OSBase + { - capabilities+: ["linux", "!e3"], - name+: "-linux", - os:: "linux", - }, - LinuxDockerAMD64Musl(defs):: self.Linux { - docker: { - image: defs.linux_docker_image_amd64_musl - }, - # No packages are need for building the musl static libs - packages: {}, - }, - - LinuxAMD64(defs, for_jdk_build):: self.Linux + self.AMD64 { - docker: { - image: defs.linux_docker_image_amd64, - mount_modules: true # needed for installing the devtoolset package below - }, - packages+: if for_jdk_build then { - # devkit_platform_revisions in make/conf/jib-profiles.js - "devkit:gcc14.2.0-OL6.4+1" : "==0" - } else { - # When building/testing GraalVM, do not use a devkit as it is known not to - # work well when dynamically linking libstdc++. - devtoolset: "==7" - }, - }, - LinuxAArch64(defs, for_jdk_build):: self.Linux + self.AArch64 { - packages+: if for_jdk_build then { - # devkit_platform_revisions in make/conf/jib-profiles.js - "devkit:gcc14.2.0-OL7.6+1" : "==0" - } else { - # See GR-26071 as well as comment in self.LinuxAMD64 - devtoolset: "==7" - }, - docker:{ - image: defs.linux_docker_image_amd64, - mount_modules: true - }, - }, - Darwin:: self.OSBase + { - jdk_home(java_home):: java_home + "/../..", - java_home(jdk_home):: jdk_home + "/Contents/Home", - packages+: { - # No need to specify a "make" package as Mac OS X has make 3.81 - # available once Xcode has been installed. - }, - os:: "darwin", - name+: "-darwin", - }, - DarwinAMD64:: self.Darwin + self.AMD64 + { - # JDK 17 switched to Xcode 12.4 which requires 10.15.4 - # at a minimum (GR-32439) - capabilities+: ["darwin_ventura"] - }, - DarwinAArch64:: self.Darwin + self.AArch64 + { - capabilities+: ["darwin_ventura"], - }, - - AMD64:: { - capabilities+: ["amd64"], - name+: "-amd64", - }, - - AMD64Musl:: self.AMD64 + { - name+: "-musl", - }, - - AArch64:: { - capabilities+: ["aarch64"], - name+: "-aarch64", - }, - - BootJDK:: { - downloads+: { - BOOT_JDK: { - name : "jpg-jdk", - version : "24", - build_id: "36", - release: true, - platformspecific: true - } - } - }, - - MuslBootJDK:: { - downloads+: { - BOOT_JDK: { - name: "labsjdk", - version: "ee-24+36-jvmci-b01-linux-amd64_musl", - platformspecific: false - } - }, - environment+: { - LD_LIBRARY_PATH: "$BOOT_JDK/lib/server" - } - }, - - JTReg:: { - downloads+: { - JT_HOME: { - name : "jtreg", - version : "4.2" - } - } - }, - - local setupJDKSources(conf) = { - run+: [ - # To reduce load, the CI system does not fetch all tags so it must - # be done explicitly as `build_labsjdk.py` relies on it. - ["git", "fetch", "-q", "--tags"], - ] + (if conf.os == "windows" then [ - # Need to fix line endings on Windows to satisfy cygwin - # https://stackoverflow.com/a/26408129 - ["set-export", "JDK_SRC_DIR", "${PWD}\\..\\jdk"], - ["git", "clone", "--quiet", "--config", "core.autocrlf=input", "-c", "gc.auto=0", ".", "${JDK_SRC_DIR}"] - ] else [ - ["set-export", "JDK_SRC_DIR", "${PWD}"] - ]) + [ - ["set-export", "JDK_SUITE_DIR", "${JDK_SRC_DIR}"] - ], - }, - - Build(defs, conf, is_musl_build):: conf + setupJDKSources(conf) + (if is_musl_build then self.MuslBootJDK else self.BootJDK) + { - name: "build-jdk" + conf.name, - timelimit: "3:00:00", # Windows is the long pole - diskspace_required: "10G", - logs: ["*.log"], - targets: ["gate"], - deploysArtifacts: true, - - local build_labsjdk(jdk_debug_level, java_home_env_var) = [ - ["set-export", java_home_env_var, conf.path("${PWD}/../%s-java-home" % jdk_debug_level)], - ["python3", "-u", conf.path("${LABSJDK_BUILDER_DIR}/build_labsjdk.py"), - "--boot-jdk=${BOOT_JDK}", - "--clean-after-build", - "--uploader=disabled", - "--jdk-debug-level=" + jdk_debug_level, - "--test=" + run_test_spec, - "--java-home-link-target=${%s}" % java_home_env_var, - ] + (if is_musl_build then ["--bundles=static-libs"] else []) - + ["${JDK_SRC_DIR}"], - (if !is_musl_build then [conf.exe("${%s}/bin/java" % java_home_env_var), "-version"] else ["echo"]) - ], - - run+: (if !is_musl_build then [ - # Checks that each devkit mentioned in this file corresponds to a devkit in make/conf/jib-profiles.js - ["python3", "-u", conf.path("${PWD}/.ci/check_devkit_versions.py")], - ] else []) + [ - ["set-export", "LABSJDK_BUILDER_DIR", conf.path("${PWD}/../labsjdk-builder")], - ["git", "clone", "--quiet", "--config", "core.autocrlf=input", defs.labsjdk_builder_url, "${LABSJDK_BUILDER_DIR}"], - ["git", "-C", "${LABSJDK_BUILDER_DIR}", "checkout", labsjdk_builder_version], - - # This restricts cygwin to be on the PATH only while using jib. - # It must not be on the PATH when building Graal. - ["set-export", "OLD_PATH", "${PATH}"], - ["set-export", "PATH", "${JIB_PATH}"], - ["set-export", "JIB_SERVER", defs.jib_server], - ["set-export", "JIB_SERVER_MIRRORS", defs.jib_server_mirrors], - ["set-export", "JIB_DATA_DIR", conf.path("${PWD}/../jib")] - ] + - build_labsjdk("release", "JAVA_HOME") + - build_labsjdk("fastdebug", "JAVA_HOME_FASTDEBUG") + - (if !is_musl_build then [ - ["set-export", "PATH", "${OLD_PATH}"], - - # Prepare for publishing - ["set-export", "JDK_HOME", conf.path("${PWD}/jdk_home")], - ["cd", "${JAVA_HOME}"], - conf.copydir(conf.jdk_home("."), "${JDK_HOME}"), - ["cd", "${JAVA_HOME_FASTDEBUG}"], - conf.copydir(conf.jdk_home("."), "${JDK_HOME}_fastdebug") - ] else []), - - }, - - # Downstream Graal branch to test against. If you change this value to anything but - # "master", you must create an ol-jira issue to change it back to master once the - # next JVMCI release has been made. Add the issue id as a comment here. - # You might want to point this to the merge commit of a Graal PR, i.e., include - # the "_gate" suffix. - local downstream_branch = "master", - - local clone_graal(defs) = { - # Checkout the graal-enterprise repo to the "_gate" version of the - # named downstream branch. This ensures the graal-enterprise and - # graal repos will be in sync. - local branch = downstream_branch, - - run+: [ - ["git", "clone", defs.graal_enterprise_url], - - # Reset mx to the version expected by Graal - ["git", "-C", "$MX_HOME", "reset", "--hard", ["python3", "-c", "import json; print(json.load(open('" + self.path("graal-enterprise/common.json") + "'))['mx_version'])"]], - - # This puts cygwin on the PATH so that `test` and `cat` are available - ["set-export", "OLD_PATH", "${PATH}"], - ["set-export", "PATH", "${JIB_PATH}"], - - # Use branch recorded by previous builder or record it now for subsequent builder(s) - ["test", "-f", "graal-enterprise.commit", "||", "echo", branch, ">graal-enterprise.commit"], - ["git", "-C", "graal-enterprise", "checkout", ["cat", "graal-enterprise.commit"]], - ["git", "-C", "graal-enterprise", "rev-list", "-n", "1", "HEAD", ">graal-enterprise.commit"], - - # Restore PATH as cygwin must not be on the PATH when building Graal. - ["set-export", "PATH", "${OLD_PATH}"], - - ["mx", "-p", "graal-enterprise/graal-enterprise", "sforceimports"] - ] - }, - - local requireLabsJDK(conf) = { - requireArtifacts+: [ - { - name: "labsjdk" + conf.name, - dir: "." - } - ], - run+: [ - ["set-export", "JAVA_HOME", conf.java_home("${PWD}/jdk_home")], - ["set-export", "JAVA_HOME_FASTDEBUG", conf.java_home("${PWD}/jdk_home_fastdebug")] - ] - }, - - CompilerTests(defs, conf, fastdebug):: conf + requireLabsJDK(conf) + clone_graal(defs) + { - name: "test-compiler" + (if fastdebug then "-fastdebug" else "") + conf.name, - timelimit: "1:45:00", - logs: ["*.log"], - targets: ["gate"], - environment+: { - # Disable JVMCI version check - JVMCI_VERSION_CHECK: "ignore" - }, - run+: if !fastdebug then [ - ["mx", "--java-home=${JAVA_HOME}", "-p", "graal-enterprise/graal-enterprise", "gate", "--tags", "build,test,bootstraplite"] - ] else [ - # Since fastdebug is slow, build with product - ["mx", "--java-home=${JAVA_HOME}", "-p", "graal/compiler", "build"], - ["mx", "--java-home=${JAVA_HOME_FASTDEBUG}", "-p", "graal/compiler", "gate", "--tags", "bootstraplite"] - ] - }, - - # Build and test JavaScript on GraalVM - JavaScriptTests(defs, conf):: conf + requireLabsJDK(conf) + clone_graal(defs) + { - local jsvm = ["mx", "-p", "graal-enterprise/vm-enterprise", - "--dynamicimports", "/graal-js,/substratevm-enterprise", - "--components=Graal.js,Native Image Enterprise", - "--native-images=js"], - - name: "test-js" + conf.name, - timelimit: "1:30:00", - logs: ["*.log"], - targets: ["gate"], - environment+: { - # Disable JVMCI version check - JVMCI_VERSION_CHECK: "ignore" - }, - run+: [ - # Build and test JavaScript on GraalVM - jsvm + ["build"], - ["set-export", "GRAALVM_HOME", jsvm + ["graalvm-home"]], - ["${GRAALVM_HOME}/bin/js", ".ci/add.js"], - ] + - if conf.os != "windows" then [ - # Native launchers do not yet support --jvm mode on Windows - ["${GRAALVM_HOME}/bin/js", "--jvm", ".ci/add.js"] - ] else [] - }, - - # Build and test LibGraal - TestLibGraal(defs, conf):: conf + requireLabsJDK(conf) + clone_graal(defs) { - name: "test-libgraal" + conf.name, - timelimit: "1:45:00", - logs: ["*.log"], - targets: ["gate"], - run+: [ - ["mx", "-p", "graal-enterprise/vm-enterprise", - "--env", "libgraal-enterprise", - "--extra-image-builder-argument=-J-esa", - "--extra-image-builder-argument=-H:+ReportExceptionStackTraces", "build"], - ["mx", "-p", "graal-enterprise/vm-enterprise", - "--env", "libgraal-enterprise", - "gate", "--task", "LibGraal"], - ], - - # Longest running tests as reported in the "LibGraal Truffle:unittest" gate task. - # We can skip these in labsjdk testing as a) they add a lot of time to the gate, - # b) will be included in the Graal integration PR and c) have not revealed any - # issues in labsjdk for a long time. - local truffle_long_running = [ - /* 384,019 ms */ "com.oracle.truffle.api.test.polyglot.PolyglotCachingTest", - /* 249,545 ms */ "org.graalvm.compiler.truffle.test.LanguageContextFreedTest", - /* 219,747 ms */ "com.oracle.truffle.api.dsl.test.WeakCachedTest", - /* 101,065 ms */ "org.graalvm.compiler.truffle.test.ExitDuringCompilationTest", - /* 83,564 ms */ "com.oracle.truffle.api.test.polyglot.ContextAPITest", - /* 75,937 ms */ "com.oracle.truffle.api.debug.test.DebuggerSessionTest", - /* 61,275 ms */ "com.oracle.truffle.api.test.polyglot.LoggingTest", - /* 52,982 ms */ "com.oracle.truffle.api.staticobject.test.InheritanceTest", - /* 47,701 ms */ "com.oracle.truffle.api.test.polyglot.ValueAPITest", - /* 31,776 ms */ "org.graalvm.compiler.truffle.test.CompilationMemoryTest" - ], - - environment+: { - # Disable JVMCI version check - JVMCI_VERSION_CHECK: "ignore", - - # The Truffle TCK tests run as a part of the Truffle TCK gate in the graal repo - TEST_LIBGRAAL_EXCLUDE: "com.oracle.truffle.tck.tests.* com.oracle.truffle.tools.* com.oracle.truffle.regex.* " + - - # Skip the longest running Truffle tests - std.join(" ", truffle_long_running) - } - }, - - local build_confs(defs) = [ - self.LinuxAMD64(defs, true), - self.LinuxAArch64(defs, true), - self.DarwinAMD64, - self.DarwinAArch64, - self.Windows + self.AMD64 - ], - - local graal_confs(defs) = [ - self.LinuxAMD64(defs, false), - self.LinuxAArch64(defs, false), - self.DarwinAMD64, - self.DarwinAArch64, - self.Windows + self.AMD64 - ], - - local amd64_musl_confs(defs) = [ - self.LinuxDockerAMD64Musl(defs) + self.AMD64Musl, - ], - - DefineBuilds(defs):: [ self.Build(defs, conf, is_musl_build=false) for conf in build_confs(defs) ] + - [ self.Build(defs, conf, is_musl_build=true) for conf in amd64_musl_confs(defs) ], - - local defs = { - labsjdk_builder_url: "", - linux_docker_image_amd64_musl: "", - linux_docker_image_amd64: "", - jib_server: "", - jib_server_mirrors: "", - graal_enterprise_url: "" - }, - - builds: self.DefineBuilds(defs) -}