diff --git a/buildkite/create_images.py b/buildkite/create_images.py index ef8f078880..4b59252ab5 100755 --- a/buildkite/create_images.py +++ b/buildkite/create_images.py @@ -29,41 +29,9 @@ DEBUG = False IMAGE_CREATION_VMS = { - # Find the newest FreeBSD 11 image via: - # gcloud compute images list --project freebsd-org-cloud-dev \ - # --no-standard-images - # ('bk-freebsd11',): { - # 'source_image': 'https://www.googleapis.com/compute/v1/projects/freebsd-org-cloud-dev/global/images/freebsd-11-1-stable-amd64-2017-12-28', - # 'scripts': [ - # 'setup-freebsd.sh', - # 'install-buildkite-agent.sh' - # ] - # }, "bk-docker": { - "project": "bazel-untrusted", - "zone": "us-central1-a", - "source_image_project": "ubuntu-os-cloud", - "source_image_family": "ubuntu-1804-lts", - "setup_script": "setup-docker.sh", - "guest_os_features": ["VIRTIO_SCSI_MULTIQUEUE"], - "licenses": [ - "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" - ], - }, - "bk-testing-docker": { - "project": "bazel-untrusted", - "zone": "us-central1-a", - "source_image_project": "ubuntu-os-cloud", - "source_image_family": "ubuntu-1804-lts", - "setup_script": "setup-docker.sh", - "guest_os_features": ["VIRTIO_SCSI_MULTIQUEUE"], - "licenses": [ - "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" - ], - }, - "bk-trusted-docker": { "project": "bazel-public", - "zone": "us-central1-a", + "zone": "us-central1-f", "source_image_project": "ubuntu-os-cloud", "source_image_family": "ubuntu-1804-lts", "setup_script": "setup-docker.sh", @@ -73,24 +41,8 @@ ], }, "bk-windows": { - "project": "bazel-untrusted", - "zone": "us-central1-a", - "source_image_project": "windows-cloud", - "source_image_family": "windows-1909-core", - "setup_script": "setup-windows.ps1", - "guest_os_features": ["VIRTIO_SCSI_MULTIQUEUE"], - }, - "bk-testing-windows": { - "project": "bazel-untrusted", - "zone": "us-central1-a", - "source_image_project": "windows-cloud", - "source_image_family": "windows-1909-core", - "setup_script": "setup-windows.ps1", - "guest_os_features": ["VIRTIO_SCSI_MULTIQUEUE"], - }, - "bk-trusted-windows": { "project": "bazel-public", - "zone": "us-central1-a", + "zone": "us-central1-f", "source_image_project": "windows-cloud", "source_image_family": "windows-1909-core", "setup_script": "setup-windows.ps1", @@ -98,7 +50,7 @@ }, "windows-playground": { "project": "di-cloud-exp", - "zone": "us-central1-a", + "zone": "us-central1-f", "network": "default", "source_image_project": "windows-cloud", "source_image_family": "windows-2019", diff --git a/buildkite/instances.yml b/buildkite/instances.yml index 875ab3ec79..bb37844ce8 100644 --- a/buildkite/instances.yml +++ b/buildkite/instances.yml @@ -14,12 +14,13 @@ default_vm: boot_disk_size: 500GB boot_disk_type: pd-ssd - machine_type: n1-highmem-32 + machine_type: c2-standard-30 network: default region: us-central1 restart-on-failure: False scopes: cloud-platform zone: us-central1-f + image_project: bazel-public tags: health-check health_check: buildkite-check initial_delay: 60 @@ -28,52 +29,38 @@ instance_groups: count: 100 project: bazel-untrusted service_account: buildkite@bazel-untrusted.iam.gserviceaccount.com - image_project: bazel-untrusted image_family: bk-docker - machine_type: c2-standard-30 metadata_from_file: startup-script=startup-docker-pdssd.sh - name: bk-testing-docker count: 30 project: bazel-untrusted service_account: buildkite-testing@bazel-untrusted.iam.gserviceaccount.com - image_project: bazel-untrusted image_family: bk-testing-docker - machine_type: c2-standard-30 metadata_from_file: startup-script=startup-docker-pdssd.sh - name: bk-trusted-docker count: 8 project: bazel-public service_account: buildkite-trusted@bazel-public.iam.gserviceaccount.com - image_project: bazel-public image_family: bk-trusted-docker - machine_type: c2-standard-30 metadata_from_file: startup-script=startup-docker-pdssd.sh - name: bk-windows count: 30 project: bazel-untrusted service_account: buildkite@bazel-untrusted.iam.gserviceaccount.com - image_project: bazel-untrusted image_family: bk-windows - local_ssd: - - interface=scsi - metadata_from_file: windows-startup-script-ps1=startup-windows.ps1 + metadata_from_file: windows-startup-script-ps1=startup-windows-pdssd.ps1 - name: bk-testing-windows count: 10 project: bazel-untrusted service_account: buildkite-testing@bazel-untrusted.iam.gserviceaccount.com - image_project: bazel-untrusted image_family: bk-testing-windows - machine_type: c2-standard-30 metadata_from_file: windows-startup-script-ps1=startup-windows-pdssd.ps1 - name: bk-trusted-windows count: 4 project: bazel-public service_account: buildkite-trusted@bazel-public.iam.gserviceaccount.com - image_project: bazel-public image_family: bk-trusted-windows - local_ssd: - - interface=scsi - metadata_from_file: windows-startup-script-ps1=startup-windows.ps1 + metadata_from_file: windows-startup-script-ps1=startup-windows-pdssd.ps1 physical_clusters: - name: buildkite-imacpro count: 20 diff --git a/buildkite/setup-windows.ps1 b/buildkite/setup-windows.ps1 index ab00f97607..8f43cd0975 100755 --- a/buildkite/setup-windows.ps1 +++ b/buildkite/setup-windows.ps1 @@ -236,80 +236,73 @@ Remove-Item "${android_sdk_root}\tools.old" -Force -Recurse & "${android_sdk_root}\tools\bin\sdkmanager.bat" "platforms;android-28" & "${android_sdk_root}\tools\bin\sdkmanager.bat" "extras;android;m2repository" -$myhostname = [System.Net.Dns]::GetHostName() -if ($myhostname -like "bk-*") { - ## Download and unpack our Git snapshot. - Write-Host "Downloading Git snapshot..." - $bazelbuild_url = "https://storage.googleapis.com/bazel-git-mirror/bazelbuild-mirror.zip" - $bazelbuild_zip = "c:\temp\bazelbuild-mirror.zip" - $bazelbuild_root = "c:\buildkite" - (New-Object Net.WebClient).DownloadFile($bazelbuild_url, $bazelbuild_zip) - Write-Host "Unpacking Git snapshot..." - Expand-Archive -LiteralPath $bazelbuild_zip -DestinationPath $bazelbuild_root -Force - Remove-Item $bazelbuild_zip - - ## Download and install the Buildkite agent. - Write-Host "Grabbing latest Buildkite Agent version number from GitHub..." - $url = "https://github.com/buildkite/agent/releases/latest" - $req = [system.Net.HttpWebRequest]::Create($url) - $res = $req.getresponse() - $res.Close() - $buildkite_agent_version = $res.ResponseUri.AbsolutePath.TrimStart("/buildkite/agent/releases/tag/v") - - Write-Host "Downloading Buildkite agent..." - $buildkite_agent_url = "https://github.com/buildkite/agent/releases/download/v${buildkite_agent_version}/buildkite-agent-windows-amd64-${buildkite_agent_version}.zip" - $buildkite_agent_zip = "c:\temp\buildkite-agent.zip" - $buildkite_agent_root = "c:\buildkite" - New-Item $buildkite_agent_root -ItemType "directory" -Force - (New-Object Net.WebClient).DownloadFile($buildkite_agent_url, $buildkite_agent_zip) - [System.IO.Compression.ZipFile]::ExtractToDirectory($buildkite_agent_zip, $buildkite_agent_root) - Remove-Item $buildkite_agent_zip - New-Item "${buildkite_agent_root}\hooks" -ItemType "directory" -Force - New-Item "${buildkite_agent_root}\plugins" -ItemType "directory" -Force - $env:PATH = [Environment]::GetEnvironmentVariable("PATH", "Machine") + ";${buildkite_agent_root}" - [Environment]::SetEnvironmentVariable("PATH", $env:PATH, "Machine") - - ## Remove empty folders (";;") from PATH. - $env:PATH = [Environment]::GetEnvironmentVariable("PATH", "Machine").replace(";;", ";") - [Environment]::SetEnvironmentVariable("PATH", $env:PATH, "Machine") - - ## Create an unprivileged user that we'll run the Buildkite agent as. - # The password used here is not relevant for security, as the server is behind a - # firewall blocking all incoming access and locally we run the CI jobs as that - # user anyway. - Write-Host "Creating Buildkite service user..." - $buildkite_username = "b" - $buildkite_password = "Bu1ldk1t3" - $buildkite_secure_password = ConvertTo-SecureString $buildkite_password -AsPlainText -Force - New-LocalUser -Name $buildkite_username -Password $buildkite_secure_password -UserMayNotChangePassword - Add-NTFSAccess -Path "C:\buildkite" -Account "b" -AccessRights FullControl - - ## Allow the Buildkite agent to store SSH host keys in this folder. - Write-Host "Creating C:\buildkite\.ssh folder..." - New-Item "c:\buildkite\.ssh" -ItemType "directory" - - Write-Host "Creating C:\buildkite\logs folder..." - New-Item "c:\buildkite\logs" -ItemType "directory" - - ## Create a service for the Buildkite agent. - & choco install nssm - - Write-Host "Creating Buildkite Agent service..." - nssm install "buildkite-agent" ` - "c:\buildkite\buildkite-agent.exe" ` - "start" - nssm set "buildkite-agent" "AppDirectory" "c:\buildkite" - nssm set "buildkite-agent" "DisplayName" "Buildkite Agent" - nssm set "buildkite-agent" "Start" "SERVICE_DEMAND_START" - nssm set "buildkite-agent" "ObjectName" ".\${buildkite_username}" "$buildkite_password" - nssm set "buildkite-agent" "AppExit" "Default" "Exit" - nssm set "buildkite-agent" "AppStdout" "COM1" - nssm set "buildkite-agent" "AppStderr" "COM1" -} else { - ## Remove empty folders (";;") from PATH. - $env:PATH = [Environment]::GetEnvironmentVariable("PATH", "Machine").replace(";;", ";") - [Environment]::SetEnvironmentVariable("PATH", $env:PATH, "Machine") -} +## Download and unpack our Git snapshot. +Write-Host "Downloading Git snapshot..." +$bazelbuild_url = "https://storage.googleapis.com/bazel-git-mirror/bazelbuild-mirror.zip" +$bazelbuild_zip = "c:\temp\bazelbuild-mirror.zip" +$bazelbuild_root = "c:\buildkite" +(New-Object Net.WebClient).DownloadFile($bazelbuild_url, $bazelbuild_zip) +Write-Host "Unpacking Git snapshot..." +Expand-Archive -LiteralPath $bazelbuild_zip -DestinationPath $bazelbuild_root -Force +Remove-Item $bazelbuild_zip + +## Download and install the Buildkite agent. +Write-Host "Grabbing latest Buildkite Agent version number from GitHub..." +$url = "https://github.com/buildkite/agent/releases/latest" +$req = [system.Net.HttpWebRequest]::Create($url) +$res = $req.getresponse() +$res.Close() +$buildkite_agent_version = $res.ResponseUri.AbsolutePath.TrimStart("/buildkite/agent/releases/tag/v") + +Write-Host "Downloading Buildkite agent..." +$buildkite_agent_url = "https://github.com/buildkite/agent/releases/download/v${buildkite_agent_version}/buildkite-agent-windows-amd64-${buildkite_agent_version}.zip" +$buildkite_agent_zip = "c:\temp\buildkite-agent.zip" +$buildkite_agent_root = "c:\buildkite" +New-Item $buildkite_agent_root -ItemType "directory" -Force +(New-Object Net.WebClient).DownloadFile($buildkite_agent_url, $buildkite_agent_zip) +[System.IO.Compression.ZipFile]::ExtractToDirectory($buildkite_agent_zip, $buildkite_agent_root) +Remove-Item $buildkite_agent_zip +New-Item "${buildkite_agent_root}\hooks" -ItemType "directory" -Force +New-Item "${buildkite_agent_root}\plugins" -ItemType "directory" -Force +$env:PATH = [Environment]::GetEnvironmentVariable("PATH", "Machine") + ";${buildkite_agent_root}" +[Environment]::SetEnvironmentVariable("PATH", $env:PATH, "Machine") + +## Remove empty folders (";;") from PATH. +$env:PATH = [Environment]::GetEnvironmentVariable("PATH", "Machine").replace(";;", ";") +[Environment]::SetEnvironmentVariable("PATH", $env:PATH, "Machine") + +## Create an unprivileged user that we'll run the Buildkite agent as. +# The password used here is not relevant for security, as the server is behind a +# firewall blocking all incoming access and locally we run the CI jobs as that +# user anyway. +Write-Host "Creating Buildkite service user..." +$buildkite_username = "b" +$buildkite_password = "Bu1ldk1t3" +$buildkite_secure_password = ConvertTo-SecureString $buildkite_password -AsPlainText -Force +New-LocalUser -Name $buildkite_username -Password $buildkite_secure_password -UserMayNotChangePassword +Add-NTFSAccess -Path "C:\buildkite" -Account "b" -AccessRights FullControl + +## Allow the Buildkite agent to store SSH host keys in this folder. +Write-Host "Creating C:\buildkite\.ssh folder..." +New-Item "c:\buildkite\.ssh" -ItemType "directory" + +Write-Host "Creating C:\buildkite\logs folder..." +New-Item "c:\buildkite\logs" -ItemType "directory" + +## Create a service for the Buildkite agent. +& choco install nssm + +Write-Host "Creating Buildkite Agent service..." +nssm install "buildkite-agent" ` + "c:\buildkite\buildkite-agent.exe" ` + "start" +nssm set "buildkite-agent" "AppDirectory" "c:\buildkite" +nssm set "buildkite-agent" "DisplayName" "Buildkite Agent" +nssm set "buildkite-agent" "Start" "SERVICE_DEMAND_START" +nssm set "buildkite-agent" "ObjectName" ".\${buildkite_username}" "$buildkite_password" +nssm set "buildkite-agent" "AppExit" "Default" "Exit" +nssm set "buildkite-agent" "AppStdout" "COM1" +nssm set "buildkite-agent" "AppStderr" "COM1" Write-Host "All done, adding GCESysprep to RunOnce and rebooting..." Set-ItemProperty "HKLM:\Software\Microsoft\Windows\CurrentVersion\RunOnce" -Name "GCESysprep" -Value "c:\Program Files\Google\Compute Engine\sysprep\gcesysprep.bat"