chore(Windows): define tags in docker compose file instead of code

Jenkinsfile

@@ -19,7 +19,7 @@ pipeline {
     }
 
     stages {
-        stage('docker-agent') {
+        stage('Pipeline') {
             matrix {
                 axes {
                     axis {

build-windows.yaml

@@ -1,6 +1,6 @@
 services:
   jdk11:
-    image: jdk11-${WINDOWS_FLAVOR}-${WINDOWS_VERSION_TAG}
+    image: ${DOCKERHUB_ORGANISATION}/${DOCKERHUB_REPO}:jdk11-${WINDOWS_FLAVOR}-${WINDOWS_VERSION_TAG}
     build:
       context: ./
       dockerfile: ./windows/${WINDOWS_FLAVOR}/Dockerfile
@@ -10,8 +10,10 @@ services:
         VERSION: ${REMOTING_VERSION}
         WINDOWS_VERSION_TAG: ${WINDOWS_VERSION_TAG}
         TOOLS_WINDOWS_VERSION: ${TOOLS_WINDOWS_VERSION}
+      tags:
+        - "${DOCKERHUB_ORGANISATION}/${DOCKERHUB_REPO}:${REMOTING_VERSION}-${BUILD_NUMBER}-jdk11-${WINDOWS_FLAVOR}-${WINDOWS_VERSION_TAG}"
   jdk17:
-    image: jdk17-${WINDOWS_FLAVOR}-${WINDOWS_VERSION_TAG}
+    image: ${DOCKERHUB_ORGANISATION}/${DOCKERHUB_REPO}:jdk17-${WINDOWS_FLAVOR}-${WINDOWS_VERSION_TAG}
     build:
       context: ./
       dockerfile: ./windows/${WINDOWS_FLAVOR}/Dockerfile
@@ -21,8 +23,12 @@ services:
         VERSION: ${REMOTING_VERSION}
         WINDOWS_VERSION_TAG: ${WINDOWS_VERSION_TAG}
         TOOLS_WINDOWS_VERSION: ${TOOLS_WINDOWS_VERSION}
+      tags:
+        - "${DOCKERHUB_ORGANISATION}/${DOCKERHUB_REPO}:${REMOTING_VERSION}-${BUILD_NUMBER}-jdk17-${WINDOWS_FLAVOR}-${WINDOWS_VERSION_TAG}"
+        - "${DOCKERHUB_ORGANISATION}/${DOCKERHUB_REPO}:${REMOTING_VERSION}-${BUILD_NUMBER}-${WINDOWS_FLAVOR}-${WINDOWS_VERSION_TAG}"
+        - "${DOCKERHUB_ORGANISATION}/${DOCKERHUB_REPO}:${WINDOWS_FLAVOR}-${WINDOWS_VERSION_TAG}"
   jdk21:
-    image: jdk21-${WINDOWS_FLAVOR}-${WINDOWS_VERSION_TAG}
+    image: ${DOCKERHUB_ORGANISATION}/${DOCKERHUB_REPO}:jdk21-${WINDOWS_FLAVOR}-${WINDOWS_VERSION_TAG}
     build:
       context: ./
       dockerfile: ./windows/${WINDOWS_FLAVOR}/Dockerfile
@@ -32,3 +38,5 @@ services:
         VERSION: ${REMOTING_VERSION}
         WINDOWS_VERSION_TAG: ${WINDOWS_VERSION_TAG}
         TOOLS_WINDOWS_VERSION: ${TOOLS_WINDOWS_VERSION}
+      tags:
+        - "${DOCKERHUB_ORGANISATION}/${DOCKERHUB_REPO}:${REMOTING_VERSION}-${BUILD_NUMBER}-jdk21-${WINDOWS_FLAVOR}-${WINDOWS_VERSION_TAG}"

build.ps1

@@ -16,8 +16,10 @@ if ($AgentType -ne '' -and $AgentType -in $AgentTypes) {
 }
 $ImageType = 'windowsservercore-ltsc2019'
 $Organisation = 'jenkins4eval'
-$AgentRepository = 'agent'
-$InboundAgentRepository = 'inbound-agent'
+$Repository = @{
+    'agent' = 'agent'
+    'inbound-agent' = 'inbound-agent'
+}
 
 if(!$DisableEnvProps) {
     Get-Content env.props | ForEach-Object {
@@ -35,11 +37,11 @@ if(![String]::IsNullOrWhiteSpace($env:DOCKERHUB_ORGANISATION)) {
 }
 
 if(![String]::IsNullOrWhiteSpace($env:DOCKERHUB_REPO_AGENT)) {
-    $AgentRepository = $env:DOCKERHUB_REPO_AGENT
+    $Repository['agent'] = $env:DOCKERHUB_REPO_AGENT
 }
 
 if(![String]::IsNullOrWhiteSpace($env:DOCKERHUB_REPO_INBOUND_AGENT)) {
-    $InboundAgentRepository = $env:DOCKERHUB_REPO_INBOUND_AGENT
+    $Repository['inbound-agent'] = $env:DOCKERHUB_REPO_INBOUND_AGENT
 }
 
 if(![String]::IsNullOrWhiteSpace($env:REMOTING_VERSION)) {
@@ -72,9 +74,10 @@ Function Test-CommandExists {
     }
 }
 
-# this is the jdk version that will be used for the 'bare tag' images, e.g., jdk17-windowsservercore-1809 -> windowsserver-1809
-$defaultJdk = '17'
+# Ensure constant env vars used in the docker compose file are defined
+$env:DOCKERHUB_ORGANISATION = "$Organisation"
 $env:REMOTING_VERSION = "$RemotingVersion"
+$env:BUILD_NUMBER = $BuildNumber
 
 $items = $ImageType.Split("-")
 $env:WINDOWS_FLAVOR = $items[0]
@@ -99,98 +102,67 @@ function Test-Image {
     $items = $AgentTypeAndImageName.Split("|")
     $agentType = $items[0]
     $imageName = $items[1]
+    $imageNameItems = $imageName.Split(":")
+    $imageTag = $imageNameItems[1]
 
-    Write-Host "= TEST: Testing ${agentType} image ${imageName}:"
+    Write-Host "= TEST: Testing ${imageName} image:"
 
-    $env:AGENT_TYPE = $agentType
-    $env:AGENT_IMAGE = $imageName
-    $env:VERSION = "$RemotingVersion-$BuildNumber"
+    $env:IMAGE_NAME = $imageName
+    $env:VERSION = "$RemotingVersion"
 
-    $targetPath = '.\target\{0}\{1}' -f $agentType, $imageName
+    $targetPath = '.\target\{0}\{1}' -f $agentType, $imageTag
     if(Test-Path $targetPath) {
         Remove-Item -Recurse -Force $targetPath
     }
     New-Item -Path $targetPath -Type Directory | Out-Null
     $configuration.Run.Path = 'tests\{0}.Tests.ps1' -f $agentType
     $configuration.TestResult.OutputPath = '{0}\junit-results.xml' -f $targetPath
     $TestResults = Invoke-Pester -Configuration $configuration
+    $failed = $false
     if ($TestResults.FailedCount -gt 0) {
-        Write-Host "There were $($TestResults.FailedCount) failed tests in ${agentType} $imageName"
-        $testFailed = $true
+        Write-Host "There were $($TestResults.FailedCount) failed tests out of $($TestResults.TotalCount) in ${imageName}"
+        $failed = $true
     } else {
-        Write-Host "There were $($TestResults.PassedCount) passed tests out of $($TestResults.TotalCount) in ${agentType} $imageName"
+        Write-Host "There were $($TestResults.PassedCount) passed tests in ${imageName}"
     }
 
-    Remove-Item env:\AGENT_TYPE
-    Remove-Item env:\AGENT_IMAGE
+    Remove-Item env:\IMAGE_NAME
     Remove-Item env:\VERSION
-}
 
-function Publish-Image {
-    param (
-        [String] $Build,
-        [String] $ImageName
-    )
-    if ($DryRun) {
-        Write-Host "= PUBLISH: (dry-run) docker tag then publish '$Build $ImageName'"
-    } else {
-        Write-Host "= PUBLISH: Tagging $Build => full name = $ImageName"
-        docker tag "$Build" "$ImageName"
-
-        Write-Host "= PUBLISH: Publishing $ImageName..."
-        docker push "$ImageName"
-    }
+    return $failed
 }
 
-
 $originalDockerComposeFile = 'build-windows.yaml'
 $finalDockerComposeFile = 'build-windows-current.yaml'
 $baseDockerCmd = 'docker-compose --file={0}' -f $finalDockerComposeFile
 $baseDockerBuildCmd = '{0} build --parallel --pull' -f $baseDockerCmd
 
 foreach($agentType in $AgentTypes) {
+    # Ensure remaining env vars used in the docker compose file are defined
     $env:AGENT_TYPE = $agentType
+    $env:DOCKERHUB_REPO = $Repository[$agentType]
 
     # Temporary docker compose file (git ignored)
     Copy-Item -Path $originalDockerComposeFile -Destination $finalDockerComposeFile
-    $repository = $InboundAgentRepository
-    # If it's a type "agent", set corresponding target and repository
+    # If it's an "agent" type, add the corresponding target
     if($agentType -eq 'agent') {
         yq '.services.[].build.target = \"agent\"' $originalDockerComposeFile | Out-File -FilePath $finalDockerComposeFile
-        $repository = $AgentRepository
     }
 
-    $builds = @{}
-
-    Invoke-Expression "$baseDockerCmd config --services" 2>$null | ForEach-Object {
-        $image = '{0}-{1}-{2}' -f $_, $env:WINDOWS_FLAVOR, $env:WINDOWS_VERSION_TAG # Ex: "jdk17-nanoserver-1809"
-
-        # Remove the 'jdk' prefix
-        $jdkMajorVersion = $_.Remove(0,3)
+    $builds = @()
 
-        $versionTag = "${RemotingVersion}-${BuildNumber}-${image}"
-        $tags = @( $image, $versionTag )
-
-        # Additional image tag without any 'jdk' prefix for the default JDK
-        $baseImage = "${env:WINDOWS_FLAVOR}-${env:WINDOWS_VERSION_TAG}"
-        if($jdkMajorVersion -eq "$defaultJdk") {
-            $tags += $baseImage
-            $tags += "${RemotingVersion}-${BuildNumber}-${baseImage}"
-        }
-
-        $builds[$image] = @{
-            'Tags' = $tags;
-        }
+    $compose = Invoke-Expression "$baseDockerCmd config --format=json" 2>$null | ConvertFrom-Json
+    foreach ($service in $compose.services.PSObject.Properties) {
+        $builds += $service.Value.image
     }
 
-    Write-Host "= PREPARE: List of $Organisation/$repository images and tags to be processed:"
-    ConvertTo-Json $builds
+    Write-Host "= PREPARE: List of $Organisation/$env:DOCKERHUB_REPO images and tags to be processed:"
+    Invoke-Expression "$baseDockerCmd config"
 
     Write-Host "= BUILD: Building all images..."
-    if ($DryRun) {
-        Write-Host "(dry-run) $baseDockerBuildCmd"
-    } else {
-        Invoke-Expression $baseDockerBuildCmd
+    switch ($DryRun) {
+        $true { Write-Host "(dry-run) $baseDockerBuildCmd" }
+        $false { Invoke-Expression $baseDockerBuildCmd }
     }
     Write-Host "= BUILD: Finished building all images."
 
@@ -204,8 +176,6 @@ foreach($agentType in $AgentTypes) {
         } else {
             Write-Host "= TEST: Starting test harness"
 
-            # Only fail the run afterwards in case of any test failures
-            $testFailed = $false
             $mod = Get-InstalledModule -Name Pester -MinimumVersion 5.3.0 -MaximumVersion 5.3.3 -ErrorAction SilentlyContinue
             if($null -eq $mod) {
                 Write-Host "= TEST: Pester 5.3.x not found: installing..."
@@ -230,10 +200,11 @@ foreach($agentType in $AgentTypes) {
             $configuration.Output.Verbosity = 'Diagnostic'
             $configuration.CodeCoverage.Enabled = $false
 
-            Write-Host "= TEST: Testing all images..."
             Write-Host "= TEST: Testing all ${agentType} images..."
-            foreach($image in $builds.Keys) {
-                Test-Image ('{0}|{1}' -f $agentType, $image)
+            # Only fail the run afterwards in case of any test failures
+            $testFailed = $false
+            foreach($image in $builds) {
+                $testFailed = $testFailed -or (Test-Image ('{0}|{1}' -f $agentType, $image))
             }
 
             # Fail if any test failures
@@ -247,20 +218,15 @@ foreach($agentType in $AgentTypes) {
     }
 
     if($target -eq "publish") {
-        # Only fail the run afterwards in case of any issues when publishing the docker images
-        $publishFailed = 0
-        foreach($b in $builds.Keys) {
-            foreach($tag in $Builds[$b]['Tags']) {
-                Publish-Image "$b" "${Organisation}/${Repository}:${tag}"
-                if($lastExitCode -ne 0) {
-                    $publishFailed = 1
-                }
-            }
+        Write-Host "= PUBLISH: push all images and tags"
+        switch($DryRun) {
+            $true { Write-Host "(dry-run) $baseDockerCmd push" }
+            $false { Invoke-Expression "$baseDockerCmd push" }
         }
 
         # Fail if any issues when publising the docker images
-        if($publishFailed -ne 0) {
-            Write-Error "Publish failed for ${Organisation}/${repository}!"
+        if($lastExitCode -ne 0) {
+            Write-Error "= PUBLISH: failed!"
             exit 1
         }
     }

tests/agent.Tests.ps1

@@ -1,11 +1,12 @@
 Import-Module -DisableNameChecking -Force $PSScriptRoot/test_helpers.psm1
 
-$global:AGENT_TYPE = Get-EnvOrDefault 'AGENT_TYPE' ''
-$global:AGENT_IMAGE = Get-EnvOrDefault 'AGENT_IMAGE' ''
+$global:IMAGE_NAME = Get-EnvOrDefault 'IMAGE_NAME' ''
 $global:VERSION = Get-EnvOrDefault 'VERSION' ''
 
-$items = $global:AGENT_IMAGE.Split("-")
+$imageItems = $global:IMAGE_NAME.Split(":")
+$GLOBAL:IMAGE_TAG = $imageItems[1]
 
+$items = $global:IMAGE_TAG.Split("-")
 # Remove the 'jdk' prefix
 $global:JAVAMAJORVERSION = $items[0].Remove(0,3)
 $global:WINDOWSFLAVOR = $items[1]
@@ -16,7 +17,7 @@ if ($items[2] -eq 'ltsc2019') {
 }
 
 # TODO: make this name unique for concurency
-$global:CONTAINERNAME = 'pester-jenkins-agent-{0}' -f $global:AGENT_IMAGE
+$global:CONTAINERNAME = 'pester-jenkins-agent-{0}' -f $global:IMAGE_TAG
 
 $global:CONTAINERSHELL="powershell.exe"
 if($global:WINDOWSFLAVOR -eq 'nanoserver') {
@@ -25,27 +26,33 @@ if($global:WINDOWSFLAVOR -eq 'nanoserver') {
 
 $global:GITLFSVERSION = '3.4.1'
 
+# # Uncomment to help debugging when working on this script
+# Write-Host "= DEBUG: global vars"
+# Get-Variable -Scope Global | ForEach-Object { Write-Host "$($_.Name) = $($_.Value)" }
+# Write-Host "= DEBUG: env vars"
+# Get-ChildItem Env: | ForEach-Object { Write-Host "$($_.Name) = $($_.Value)" }
+
 Cleanup($global:CONTAINERNAME)
 
-Describe "[$global:AGENT_TYPE > $global:AGENT_IMAGE] image is present" {
-    It 'builds image' {
-        $exitCode, $stdout, $stderr = Run-Program 'docker' "inspect $global:AGENT_IMAGE"
+Describe "[$global:IMAGE_NAME] image is present" {
+    It 'inspects image' {
+        $exitCode, $stdout, $stderr = Run-Program 'docker' "inspect $global:IMAGE_NAME"
         $exitCode | Should -Be 0
     }
 }
 
-Describe "[$global:AGENT_TYPE > $global:AGENT_IMAGE] correct image metadata" {
+Describe "[$global:IMAGE_NAME] correct image metadata" {
     It 'has correct volumes' {
-        $exitCode, $stdout, $stderr = Run-Program 'docker' "inspect --format='{{.Config.Volumes}}' $global:AGENT_IMAGE"
+        $exitCode, $stdout, $stderr = Run-Program 'docker' "inspect --format='{{.Config.Volumes}}' $global:IMAGE_NAME"
         $stdout = $stdout.Trim()
         $stdout | Should -Match 'C:/Users/jenkins/.jenkins'
         $stdout | Should -Match 'C:/Users/jenkins/Work'
     }
 }
 
-Describe "[$global:AGENT_TYPE > $global:AGENT_IMAGE] image has correct applications in the PATH" {
+Describe "[$global:IMAGE_NAME] image has correct applications in the PATH" {
     BeforeAll {
-        docker run --detach --interactive --tty --name "$global:CONTAINERNAME" "$global:AGENT_IMAGE" "$global:CONTAINERSHELL"
+        docker run --detach --interactive --tty --name "$global:CONTAINERNAME" "$global:IMAGE_NAME" "$global:CONTAINERSHELL"
         Is-ContainerRunning $global:CONTAINERNAME | Should -BeTrue
     }
 
@@ -88,9 +95,9 @@ Describe "[$global:AGENT_TYPE > $global:AGENT_IMAGE] image has correct applicati
     }
 }
 
-Describe "[$global:AGENT_TYPE > $global:AGENT_IMAGE] check user account" {
+Describe "[$global:IMAGE_NAME] check user account" {
     BeforeAll {
-        docker run -d -it --name "$global:CONTAINERNAME" -P "$global:AGENT_IMAGE" "$global:CONTAINERSHELL"
+        docker run -d -it --name "$global:CONTAINERNAME" -P "$global:IMAGE_NAME" "$global:CONTAINERSHELL"
         Is-ContainerRunning $global:CONTAINERNAME | Should -BeTrue
     }
 
@@ -109,9 +116,9 @@ Describe "[$global:AGENT_TYPE > $global:AGENT_IMAGE] check user account" {
     }
 }
 
-Describe "[$global:AGENT_TYPE > $global:AGENT_IMAGE] check user access to directories" {
+Describe "[$global:IMAGE_NAME] check user access to directories" {
     BeforeAll {
-        docker run -d -it --name "$global:CONTAINERNAME" -P "$global:AGENT_IMAGE" "$global:CONTAINERSHELL"
+        docker run -d -it --name "$global:CONTAINERNAME" -P "$global:IMAGE_NAME" "$global:CONTAINERSHELL"
         Is-ContainerRunning $global:CONTAINERNAME | Should -BeTrue
     }
 
@@ -139,14 +146,14 @@ $global:TEST_VERSION="4.0"
 $global:TEST_USER="test-user"
 $global:TEST_AGENT_WORKDIR="C:/test-user/something"
 
-Describe "[$global:AGENT_TYPE > $global:AGENT_IMAGE] can be built with custom build arguments" {
+Describe "[$global:IMAGE_NAME] can be built with custom build arguments" {
     BeforeAll {
         Push-Location -StackName 'agent' -Path "$PSScriptRoot/.."
 
-        $exitCode, $stdout, $stderr = Run-Program 'docker' "build --target agent --build-arg `"VERSION=${global:TEST_VERSION}`" --build-arg `"WINDOWS_VERSION_TAG=${global:WINDOWSVERSIONTAG}`" --build-arg `"TOOLS_WINDOWS_VERSION=${global:WINDOWSVERSIONFALLBACKTAG}`" --build-arg `"user=${global:TEST_USER}`" --build-arg `"AGENT_WORKDIR=${global:TEST_AGENT_WORKDIR}`" --tag ${global:AGENT_IMAGE} --file ./windows/${global:WINDOWSFLAVOR}/Dockerfile ."
+        $exitCode, $stdout, $stderr = Run-Program 'docker' "build --target agent --build-arg `"VERSION=${global:TEST_VERSION}`" --build-arg `"WINDOWS_VERSION_TAG=${global:WINDOWSVERSIONTAG}`" --build-arg `"TOOLS_WINDOWS_VERSION=${global:WINDOWSVERSIONFALLBACKTAG}`" --build-arg `"user=${global:TEST_USER}`" --build-arg `"AGENT_WORKDIR=${global:TEST_AGENT_WORKDIR}`" --tag ${global:IMAGE_NAME} --file ./windows/${global:WINDOWSFLAVOR}/Dockerfile ."
         $exitCode | Should -Be 0
 
-        $exitCode, $stdout, $stderr = Run-Program 'docker' "run -d -it --name $global:CONTAINERNAME -P $global:AGENT_IMAGE $global:CONTAINERSHELL"
+        $exitCode, $stdout, $stderr = Run-Program 'docker' "run -d -it --name $global:CONTAINERNAME -P $global:IMAGE_NAME $global:CONTAINERSHELL"
         Is-ContainerRunning $global:CONTAINERNAME | Should -BeTrue
     }
 
@@ -184,7 +191,7 @@ Describe "[$global:AGENT_TYPE > $global:AGENT_IMAGE] can be built with custom bu
     }
 
     It 'version in docker metadata' {
-        $exitCode, $stdout, $stderr = Run-Program 'docker' "inspect --format=`"{{index .Config.Labels \`"org.opencontainers.image.version\`"}}`" $global:AGENT_IMAGE"
+        $exitCode, $stdout, $stderr = Run-Program 'docker' "inspect --format=`"{{index .Config.Labels \`"org.opencontainers.image.version\`"}}`" $global:IMAGE_NAME"
         $exitCode | Should -Be 0
         $stdout.Trim() | Should -Match $global:sTEST_VERSION
     }