From 0dc7e3b7fa56e60f38c87dae02078d6424e8412b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Mon, 16 May 2022 21:35:56 +0200 Subject: [PATCH] [release/6.0] Docker fixes for the enterprise linux testing pipeline (#68913) * attempt to fix up EnterpriseTests (#65981) * attemp to fix up EnterpriseTests * fix text * increase timoeout on build step * fix temeout * add jobs * update job * fix formating * fix name * update server check * restore original resolver before tests * fix path * experiment (cherry picked from commit 9174037cc6b58de847c9c88b7dcceef2b4fda440) * upgrade docker to get clang9 to prevent coreclr crashes (#68311) (cherry picked from commit edb9c498889b85ba2be015afca2f2cc73c30355c) * Use Docker images from mcr.microsoft.com in EnterpriseTests (#68875) We started getting warnings in the build about using images from docker directly, see https://docs.opensource.microsoft.com/tools/nuget_security_analysis/container_registry_analysis/ The image from dotnet-buildtools-prereqs-docker can be used instead. (cherry picked from commit bafae1b8e6f1c6ce9798acf013b015a60def3381) Co-authored-by: Tomas Weinfurt --- eng/pipelines/libraries/enterprise/linux.yml | 85 ++++++++++--------- .../setup/apacheweb/Dockerfile | 2 +- .../EnterpriseTests/setup/docker-compose.yml | 3 + .../Net/EnterpriseTests/setup/kdc/Dockerfile | 2 +- .../setup/linuxclient/Dockerfile | 2 +- .../setup/linuxclient/test-webserver.sh | 3 + 6 files changed, 54 insertions(+), 43 deletions(-) diff --git a/eng/pipelines/libraries/enterprise/linux.yml b/eng/pipelines/libraries/enterprise/linux.yml index 904dc4c87c73c..f395c9814308f 100644 --- a/eng/pipelines/libraries/enterprise/linux.yml +++ b/eng/pipelines/libraries/enterprise/linux.yml @@ -18,10 +18,6 @@ pr: - src/libraries/System.Net.Http/* - src/libraries/System.Net.Security/* -pool: - name: NetCore1ESPool-Svc-Public - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - variables: - template: ../variables.yml - name: enterpriseTestsSetup @@ -31,45 +27,54 @@ variables: - name: containerLibrariesRoot value: /repo/src/libraries -steps: -- bash: | - cd $(enterpriseTestsSetup) - docker-compose build - displayName: Build test machine images - env: - DOTNET_RUNTIME_REPO_ROOT: $(Build.SourcesDirectory) +jobs: +- job: EnterpriseLinuxTests + timeoutInMinutes: 120 + pool: + name: NetCore1ESPool-Svc-Public + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + steps: + - bash: | + cd $(enterpriseTestsSetup) + docker-compose build + displayName: Build test machine images + env: + DOTNET_RUNTIME_REPO_ROOT: $(Build.SourcesDirectory) -- bash: | - cd $(enterpriseTestsSetup) - docker-compose up -d - displayName: Start test network and machines - env: - DOTNET_RUNTIME_REPO_ROOT: $(Build.SourcesDirectory) + - bash: | + cd $(enterpriseTestsSetup) + docker-compose up -d + displayName: Start test network and machines + env: + DOTNET_RUNTIME_REPO_ROOT: $(Build.SourcesDirectory) -- bash: | - docker exec linuxclient bash /setup/test-webserver.sh - displayName: Test linuxclient connection to web server + - bash: | + docker exec linuxclient bash /setup/test-webserver.sh + displayName: Test linuxclient connection to web server -- bash: | - docker exec linuxclient bash -c '/repo/build.sh -subset clr+libs -runtimeconfiguration release -ci /p:NoPgoOptimize=true' - displayName: Build product sources + - bash: | + docker exec linuxclient bash -c '/repo/build.sh -subset clr+libs -runtimeconfiguration release -ci /p:NoPgoOptimize=true' + docker exec linuxclient bash -c '/repo/dotnet.sh build $(containerLibrariesRoot)/System.Net.Http/tests/EnterpriseTests/System.Net.Http.Enterprise.Tests.csproj' + docker exec linuxclient bash -c '/repo/dotnet.sh build $(containerLibrariesRoot)/System.Net.Security/tests/EnterpriseTests/System.Net.Security.Enterprise.Tests.csproj' + displayName: Build product sources -- bash: | - docker exec linuxclient $(containerRunTestsCommand) $(containerLibrariesRoot)/System.Net.Http/tests/EnterpriseTests/System.Net.Http.Enterprise.Tests.csproj - docker exec linuxclient $(containerRunTestsCommand) $(containerLibrariesRoot)/System.Net.Security/tests/EnterpriseTests/System.Net.Security.Enterprise.Tests.csproj - displayName: Build and run tests + - bash: | + docker exec linuxclient bash -c 'if [ -f /erc/resolv.conf.ORI ]; then cp -f /erc/resolv.conf.ORI /etc/resolv.conf; fi' + docker exec linuxclient $(containerRunTestsCommand) $(containerLibrariesRoot)/System.Net.Http/tests/EnterpriseTests/System.Net.Http.Enterprise.Tests.csproj + docker exec linuxclient $(containerRunTestsCommand) $(containerLibrariesRoot)/System.Net.Security/tests/EnterpriseTests/System.Net.Security.Enterprise.Tests.csproj + displayName: Build and run tests -- bash: | - cd $(enterpriseTestsSetup) - docker-compose down - displayName: Stop test network and machines - env: - DOTNET_RUNTIME_REPO_ROOT: $(Build.SourcesDirectory) + - bash: | + cd $(enterpriseTestsSetup) + docker-compose down + displayName: Stop test network and machines + env: + DOTNET_RUNTIME_REPO_ROOT: $(Build.SourcesDirectory) -- task: PublishTestResults@2 - inputs: - testRunner: 'xUnit' - testResultsFiles: '**/testResults.xml' - testRunTitle: 'Enterprise Tests' - mergeTestResults: true - failTaskOnFailedTests: true + - task: PublishTestResults@2 + inputs: + testRunner: 'xUnit' + testResultsFiles: '**/testResults.xml' + testRunTitle: 'Enterprise Tests' + mergeTestResults: true + failTaskOnFailedTests: true diff --git a/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/apacheweb/Dockerfile b/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/apacheweb/Dockerfile index 0b6a7b19eba88..d3e266f4f7810 100644 --- a/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/apacheweb/Dockerfile +++ b/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/apacheweb/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20220421022739-9c434db ARG DEBIAN_FRONTEND=noninteractive diff --git a/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/docker-compose.yml b/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/docker-compose.yml index c54adfb7caaea..0b524f35b381f 100644 --- a/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/docker-compose.yml +++ b/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/docker-compose.yml @@ -60,6 +60,9 @@ services: hostname: linuxclient domainname: linux.contoso.com dns_search: linux.contoso.com + privileged: true + dns: + - 8.8.8.8 volumes: - shared-volume:/SHARED - ${DOTNET_RUNTIME_REPO_ROOT}:/repo diff --git a/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/kdc/Dockerfile b/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/kdc/Dockerfile index 1b28145ff833b..9abd3fa729d3a 100644 --- a/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/kdc/Dockerfile +++ b/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/kdc/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20220421022739-9c434db COPY ./kdc/kadm5.acl /etc/krb5kdc/ COPY ./kdc/kdc.conf /etc/krb5kdc/ diff --git a/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/linuxclient/Dockerfile b/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/linuxclient/Dockerfile index 853582e7bcf3c..7b7afc1b4a9b9 100644 --- a/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/linuxclient/Dockerfile +++ b/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/linuxclient/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20211022152710-047508b +FROM mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-20220421022739-9c434db # Prevents dialog prompting when installing packages ARG DEBIAN_FRONTEND=noninteractive diff --git a/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/linuxclient/test-webserver.sh b/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/linuxclient/test-webserver.sh index 2afd01f34335c..058604296dc3b 100644 --- a/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/linuxclient/test-webserver.sh +++ b/src/libraries/Common/tests/System/Net/EnterpriseTests/setup/linuxclient/test-webserver.sh @@ -4,3 +4,6 @@ kdestroy echo password | kinit user1 curl --verbose --negotiate -u: http://apacheweb.linux.contoso.com kdestroy + +nslookup github.com +