[7.17] [buildkite] Add Windows packaging and platform support tests t…

…o periodic pipeline (#98072) (#98329) * [buildkite] Add Windows packaging and platform support tests to periodic pipeline (#98072) (cherry picked from commit dfadca8)
elastic · Aug 28, 2023 · a950367 · a950367
1 parent 97d53db
commit a950367
Show file tree

Hide file tree

Showing 8 changed files with 299 additions and 25 deletions.
diff --git a/.buildkite/hooks/pre-command.bat b/.buildkite/hooks/pre-command.bat
@@ -0,0 +1,22 @@
+@ECHO OFF
+
+FOR /F "tokens=* eol=#" %%i in ('type .ci\java-versions.properties') do set %%i
+
+SET JAVA_HOME=%USERPROFILE%\.java\%ES_BUILD_JAVA%
+SET JAVA11_HOME=%USERPROFILE%\.java\java11
+SET JAVA16_HOME=%USERPROFILE%\.java\openjdk16
+
+SET GRADLEW=./gradlew --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+SET GRADLEW_BAT=./gradlew.bat --parallel --no-daemon --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+
+(if not exist "%USERPROFILE%/.gradle" mkdir "%USERPROFILE%/.gradle") && (echo. >> "%USERPROFILE%/.gradle/gradle.properties" && echo org.gradle.daemon=false >> "%USERPROFILE%/.gradle/gradle.properties")
+
+set WORKSPACE=%cd%
+set BUILD_NUMBER=%BUILDKITE_BUILD_NUMBER%
+set COMPOSE_HTTP_TIMEOUT=120
+set JOB_BRANCH=%BUILDKITE_BRANCH%
+
+set GRADLE_BUILD_CACHE_USERNAME=vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
+set GRADLE_BUILD_CACHE_PASSWORD=vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache
+
+exit /b 0
diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -45,3 +45,46 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+  - group: packaging-tests-windows
+    steps:
+      - label: "{{matrix.image}} / packaging-tests-windows"
+        command: |
+          .\.buildkite\scripts\run-script.ps1 .\.ci\scripts\packaging-test.ps1
+        timeout_in_minutes: 180
+        matrix:
+          setup:
+            image:
+              - windows-2016
+              - windows-2019
+              - windows-2022
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-32-98304
+          diskType: pd-ssd
+          diskSizeGb: 350
+        env: {}
+  - group: platform-support-windows
+    steps:
+      - label: "{{matrix.image}} / {{matrix.GRADLE_TASK}} / platform-support-windows"
+        command: |
+          .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
+        timeout_in_minutes: 360
+        matrix:
+          setup:
+            image:
+              - windows-2016
+              - windows-2019
+              - windows-2022
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - bwcTestSnapshots
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-32-98304
+          diskType: pd-ssd
+          diskSizeGb: 350
+        env:
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
diff --git a/.buildkite/scripts/run-script.ps1 b/.buildkite/scripts/run-script.ps1
@@ -0,0 +1,154 @@
+# Usage: .buildkite/scripts/run-script.ps1 <script-or-simple-command>
+# Example: .buildkite/scripts/run-script.ps1 bash .buildkite/scripts/tests.sh
+# Example: .buildkite/scripts/run-script.ps1 .buildkite/scripts/other-tests.ps1
+#
+# NOTE: Apparently passing arguments in powershell is a nightmare, so you shouldn't do it unless it's really simple. Just use the wrapper to call a script instead.
+# See: https://stackoverflow.com/questions/6714165/powershell-stripping-double-quotes-from-command-line-arguments
+# and: https://github.com/PowerShell/PowerShell/issues/3223#issuecomment-487975049
+#
+# See here: https://github.com/buildkite/agent/issues/2202
+# Background processes after the buildkite step script finishes causes the job to hang.
+# So, until this is fixed/changed in buildkite-agent (if ever), we can use this wrapper.
+
+# This wrapper:
+# - Creates a Windows job object (which is like a process group)
+# - Sets JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, which means that when the job object is closed, all processes in the job object are killed
+# - Starts running your given script, and assigns it to the job object
+# - Now, any child processes created by your script will also end up in the job object
+# - Waits for your script (and only your script, not child processes) to finish
+# - Closes the job object, which kills all processes in the job object (including any leftover child processes)
+# - Exits with the exit status from your script
+
+Add-Type -TypeDefinition @'
+using Microsoft.Win32.SafeHandles;
+using System;
+using System.ComponentModel;
+using System.Runtime.InteropServices;
+
+public class NativeMethods
+{
+    public enum JOBOBJECTINFOCLASS
+    {
+        AssociateCompletionPortInformation = 7,
+        BasicLimitInformation = 2,
+        BasicUIRestrictions = 4,
+        EndOfJobTimeInformation = 6,
+        ExtendedLimitInformation = 9,
+        SecurityLimitInformation = 5,
+        GroupInformation = 11
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    struct JOBOBJECT_BASIC_LIMIT_INFORMATION
+    {
+        public Int64 PerProcessUserTimeLimit;
+        public Int64 PerJobUserTimeLimit;
+        public UInt32 LimitFlags;
+        public UIntPtr MinimumWorkingSetSize;
+        public UIntPtr MaximumWorkingSetSize;
+        public UInt32 ActiveProcessLimit;
+        public Int64 Affinity;
+        public UInt32 PriorityClass;
+        public UInt32 SchedulingClass;
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    struct IO_COUNTERS
+    {
+        public UInt64 ReadOperationCount;
+        public UInt64 WriteOperationCount;
+        public UInt64 OtherOperationCount;
+        public UInt64 ReadTransferCount;
+        public UInt64 WriteTransferCount;
+        public UInt64 OtherTransferCount;
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION
+    {
+        public JOBOBJECT_BASIC_LIMIT_INFORMATION BasicLimitInformation;
+        public IO_COUNTERS IoInfo;
+        public UIntPtr ProcessMemoryLimit;
+        public UIntPtr JobMemoryLimit;
+        public UIntPtr PeakProcessMemoryUsed;
+        public UIntPtr PeakJobMemoryUsed;
+    }
+
+    [DllImport("Kernel32.dll", EntryPoint = "AssignProcessToJobObject", SetLastError = true)]
+    private static extern bool NativeAssignProcessToJobObject(SafeHandle hJob, SafeHandle hProcess);
+
+    public static void AssignProcessToJobObject(SafeHandle job, SafeHandle process)
+    {
+        if (!NativeAssignProcessToJobObject(job, process))
+            throw new Win32Exception();
+    }
+
+    [DllImport(
+        "Kernel32.dll",
+        CharSet = CharSet.Unicode,
+        EntryPoint = "CreateJobObjectW",
+        SetLastError = true
+    )]
+    private static extern SafeFileHandle NativeCreateJobObjectW(
+        IntPtr lpJobAttributes,
+        string lpName
+    );
+
+    [DllImport("Kernel32.dll", EntryPoint = "CloseHandle", SetLastError = true)]
+    private static extern bool NativeCloseHandle(SafeHandle hJob);
+
+    [DllImport("kernel32.dll")]
+    public static extern bool SetInformationJobObject(
+        SafeHandle hJob,
+        JOBOBJECTINFOCLASS JobObjectInfoClass,
+        IntPtr lpJobObjectInfo,
+        uint cbJobObjectInfoLength
+    );
+
+    private const UInt32 JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000;
+
+    public static SafeHandle CreateJobObjectW(string name)
+    {
+        SafeHandle job = NativeCreateJobObjectW(IntPtr.Zero, name);
+        JOBOBJECT_BASIC_LIMIT_INFORMATION info = new JOBOBJECT_BASIC_LIMIT_INFORMATION();
+        info.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
+        JOBOBJECT_EXTENDED_LIMIT_INFORMATION extendedInfo =
+            new JOBOBJECT_EXTENDED_LIMIT_INFORMATION();
+        extendedInfo.BasicLimitInformation = info;
+        int length = Marshal.SizeOf(typeof(JOBOBJECT_EXTENDED_LIMIT_INFORMATION));
+        IntPtr extendedInfoPtr = Marshal.AllocHGlobal(length);
+        Marshal.StructureToPtr(extendedInfo, extendedInfoPtr, false);
+        SetInformationJobObject(
+            job,
+            JOBOBJECTINFOCLASS.ExtendedLimitInformation,
+            extendedInfoPtr,
+            (uint)length
+        );
+        if (job.IsInvalid)
+            throw new Win32Exception();
+        return job;
+    }
+
+    public static void CloseHandle(SafeHandle job)
+    {
+        if (!NativeCloseHandle(job))
+            throw new Win32Exception();
+    }
+}
+
+'@
+
+$guid = [guid]::NewGuid().Guid
+Write-Output "Creating job object with name $guid"
+$job = [NativeMethods]::CreateJobObjectW($guid)
+$process = Start-Process -PassThru -NoNewWindow powershell -ArgumentList "$args"
+[NativeMethods]::AssignProcessToJobObject($job, $process.SafeHandle)
+
+try {
+  Write-Output "Waiting for process $($process.Id) to complete..."
+  $process | Wait-Process
+  Write-Output "Process finished with exit code $($process.ExitCode), terminating job and exiting..."
+} finally {
+  [NativeMethods]::CloseHandle($job)
+  exit $process.ExitCode
+}
diff --git a/.buildkite/scripts/windows-run-gradle.sh b/.buildkite/scripts/windows-run-gradle.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euo pipefail
+
+.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $GRADLE_TASK
diff --git a/.ci/scripts/packaging-test.ps1 b/.ci/scripts/packaging-test.ps1
@@ -12,21 +12,22 @@ $env:ES_BUILD_JAVA=$AppProps.ES_BUILD_JAVA
 $env:JAVA_TOOL_OPTIONS=''
 
 $ErrorActionPreference="Stop"
-$gradleInit = "C:\Users\$env:username\.gradle\init.d\"
+$gradleInit = "$env:USERPROFILE\.gradle\init.d\"
 echo "Remove $gradleInit"
 Remove-Item -Recurse -Force $gradleInit -ErrorAction Ignore
 New-Item -ItemType directory -Path $gradleInit
 echo "Copy .ci/init.gradle to $gradleInit"
 Copy-Item .ci/init.gradle -Destination $gradleInit
 
 [Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine")
-$env:PATH="C:\Users\jenkins\.java\$env:ES_BUILD_JAVA\bin\;$env:PATH"
+$env:PATH="$env:USERPROFILE\.java\$env:ES_BUILD_JAVA\bin\;$env:PATH"
 $env:JAVA_HOME=$null
-$env:SYSTEM_JAVA_HOME="C:\Users\jenkins\.java\java8"
+$env:SYSTEM_JAVA_HOME="$env:USERPROFILE\.java\java8"
 Remove-Item -Recurse -Force \tmp -ErrorAction Ignore
 New-Item -ItemType directory -Path \tmp
 
 $ErrorActionPreference="Continue"
-& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks
+Add-Content -Path $profile.AllUsersAllHosts -Value '$ErrorActionPreference="Continue"'
+& .\gradlew.bat -g "$env:USERPROFILE\.gradle" --parallel --no-daemon --scan --console=plain $GradleTasks
 
 exit $LastExitCode
diff --git a/.ci/scripts/run-gradle.sh b/.ci/scripts/run-gradle.sh
@@ -1,28 +1,35 @@
 #!/bin/bash
-# drop page cache and kernel slab objects on linux
-[[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
 
 rm -Rfv ~/.gradle/init.d
 mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d
-if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
-   MAX_WORKERS=16
-elif [ -f /proc/cpuinfo ]; then
-   MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
-else
-   if [[ "$OSTYPE" == "darwin"* ]]; then
-      MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
-   else
-      echo "Unsupported OS Type: $OSTYPE"
-      exit 1
-   fi
-fi
-if pwd | grep -v -q ^/dev/shm ; then
-   echo "Not running on a ramdisk, reducing number of workers"
-   MAX_WORKERS=$(($MAX_WORKERS*2/3))
-fi
 
-# Export glibc version as environment variable since some BWC tests are incompatible with later versions
-export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
+MAX_WORKERS=4
+
+# Don't run this stuff on Windows
+if ! uname -a | grep -q MING; then
+  # drop page cache and kernel slab objects on linux
+  [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
+
+  if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
+    MAX_WORKERS=16
+  elif [ -f /proc/cpuinfo ]; then
+    MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo  | uniq | sed 's/\s\+//g' |  cut -d':' -f 2`
+  else
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'`
+    else
+        echo "Unsupported OS Type: $OSTYPE"
+        exit 1
+    fi
+  fi
+  if pwd | grep -v -q ^/dev/shm ; then
+    echo "Not running on a ramdisk, reducing number of workers"
+    MAX_WORKERS=$(($MAX_WORKERS*2/3))
+  fi
+
+  # Export glibc version as environment variable since some BWC tests are incompatible with later versions
+  export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
+fi
 
 set -e
 $GRADLEW -S --max-workers=$MAX_WORKERS $@
diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle
@@ -17,6 +17,7 @@ long startTime = project.gradle.services.get(BuildRequestMetaData).getStartTime(
 
 buildScan {
   URL jenkinsUrl = System.getenv('JENKINS_URL') ? new URL(System.getenv('JENKINS_URL')) : null
+  String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') ? System.getenv('BUILDKITE_BUILD_URL') : null
 
   // Automatically publish scans from Elasticsearch CI
   if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') {
@@ -98,6 +99,47 @@ buildScan {
         value 'Git Commit ID', BuildParams.gitRevision
         link 'Source', "https://github.com/elastic/elasticsearch/tree/${BuildParams.gitRevision}"
       }
+    } else if (buildKiteUrl) { //Buildkite-specific build scan metadata
+      // Disable async upload in CI to ensure scan upload completes before CI agent is terminated
+      uploadInBackground = false
+
+      def branch = System.getenv('BUILDKITE_BRANCH')
+      def repoMatcher = System.getenv('BUILDKITE_REPO') =~ /(https:\/\/github\.com\/|git@github\.com:)(\S+)\.git/
+      def repository = repoMatcher.matches() ? repoMatcher.group(2) : "<unknown>"
+      tag 'CI'
+      link 'CI Build', buildKiteUrl
+      value 'Job Number', System.getenv('BUILDKITE_BUILD_NUMBER')
+
+
+      // Add SCM information
+      def prId = System.getenv('BUILDKITE_PULL_REQUEST')
+      if (prId != 'false') {
+        def prBaseUrl = (System.getenv('BUILDKITE_PULL_REQUEST_REPO') - ".git")
+        value 'Git Commit ID', System.getenv('BUILDKITE_COMMIT')
+        tag "pr/${prId}"
+        tag 'pull-request'
+        link 'Source', "${prBaseUrl}/tree/${System.getenv('BUILDKITE_COMMIT')}"
+        link 'Pull Request', "https://github.com/${repository}/pull/${prId}"
+      } else {
+        value 'Git Commit ID', BuildParams.gitRevision
+        link 'Source', "https://github.com/${repository}/tree/${BuildParams.gitRevision}"
+        tag branch
+      }
+
+      buildScanPublished { scan ->
+        // Attach build scan link as build metadata
+        // See: https://buildkite.com/docs/pipelines/build-meta-data
+        new ProcessBuilder('buildkite-agent', 'meta-data', 'set', "build-scan-${System.getenv('BUILDKITE_JOB_ID')}", "${scan.buildScanUri}")
+          .start()
+          .waitFor()
+
+        // Add a build annotation
+        // See: https://buildkite.com/docs/agent/v3/cli-annotate
+        def body = """<div class="mb3"><span class="p1 border rounded">${System.getenv('BUILDKITE_LABEL')}</span> :gradle: build ran: <a href="${scan.buildScanUri}"><code>gradle ${gradle.startParameter.taskNames.join(' ')}</code></a></div>"""
+        new ProcessBuilder('buildkite-agent', 'annotate', '--context', 'gradle-build-scans', '--append', '--style', 'info', body)
+          .start()
+          .waitFor()
+      }
     } else {
       tag 'LOCAL'
     }

diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java b/build-tools/src/main/java/org/elasticsearch/gradle/ReaperService.java
@@ -41,7 +41,7 @@ public abstract class ReaperService implements BuildService<ReaperService.Params
      */
     public void registerPid(String serviceId, long pid) {
         String[] killPidCommand = OS.<String[]>conditional()
-            .onWindows(() -> new String[] { "Taskkill", "/F", "/PID", String.valueOf(pid) })
+            .onWindows(() -> new String[] { "Taskkill", "/F", "/T", "/PID", String.valueOf(pid) })
             .onUnix(() -> new String[] { "kill", "-9", String.valueOf(pid) })
             .supply();
         registerCommand(serviceId, killPidCommand);