diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 2d9e5418..00000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -/testdata/** eol=lf diff --git a/.taskcluster.yml b/.taskcluster.yml index e5c4fa3b..ba9f0218 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -1,783 +1,73 @@ -# The version is always required -version: 0 -allowPullRequests: public -# Top level metadata is always required -metadata: - name: "Taskcluster Generic Worker Tests" - description: "These tests should ensure that any new commits against generic worker codebase are tested across all supported worker types in gekco." - owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here - source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here +version: 1 +policy: + pullRequests: public tasks: - - - ########################################################## - ######### Windows Server 2012 R2 Multiuser Build ######### - ########################################################## - - - provisionerId: pmoore-test - workerType: win2012r2-cu - metadata: - name: "Build/test 64 bit generic-worker (multiuser engine) on Windows Server 2012 R2" - description: "This builds and tests the 64 bit Windows version of generic-worker (multiuser engine) on Windows Server 2012 R2" - owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here - source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here - extra: - github: - # Events that will trigger this task - events: - - pull_request.opened - - pull_request.synchronize - - push - scopes: - - generic-worker:cache:generic-worker-checkout - - secrets:get:repo:github.com/taskcluster/generic-worker - payload: - features: - taskclusterProxy: true - maxRunTime: 3600 - command: - - >- - C:\cygwin\bin\wget.exe -q -O- - %TASKCLUSTER_PROXY_URL%/secrets/v1/secret/repo:github.com/taskcluster/generic-worker - | C:\cygwin\bin\sed.exe -n 's/.*"b64_encoded_credentials_batch_script": - "\(.*\)".*/\1/p' | C:\cygwin\bin\base64.exe -d > tc-creds.bat - - call tc-creds.bat 2>&1 - - set CGO_ENABLED=0 - - set GOPATH=%CD%\gopath1.10.8 - - set GOROOT=%CD%\go1.10.8\go - - set PATH=%CD%\git\cmd;%GOPATH%\bin;%GOROOT%\bin;%PATH% - - git config --global core.autocrlf false - - go version - - go env - - 'if not exist "%GOPATH%\src\github.com\taskcluster" mkdir "%GOPATH%\src\github.com\taskcluster"' - - 'cd "%GOPATH%\src\github.com\taskcluster"' - - 'if not exist generic-worker git clone {{ event.head.repo.url }} generic-worker' - - 'cd generic-worker' - - 'git fetch "{{ event.head.repo.url }}" "+{{ event.head.ref }}:refs/heads/X%TASK_ID%"' - - 'git checkout -f "X%TASK_ID%"' - - 'git reset --hard "{{ event.head.sha }}"' - - 'git clean -fdx' - - 'git checkout -B tmp -t "X%TASK_ID%"' - - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - - cd gw-codegen - - go get -v - - cd .. - - go generate - - | - :: this counts the number of lines returned by git status - :: dump temp file a directory higher, otherwise git status reports the tmp1.txt file! - git status --porcelain | C:\Windows\System32\find.exe /v /c "" > ..\tmp1.txt - set /P lines=<..\tmp1.txt - :: this checks that if more than 0 lines are returned, we fail - if %lines% gtr 0 exit /b 64 - :: find.exe will have exited with exit code 1, so need to explicitly exit with 0 - exit /b 0 - - git rev-parse HEAD > revision.txt - - set /p revision=< revision.txt - - go install -tags multiuser -v -ldflags "-X main.revision=%revision%" ./... - - set CGO_ENABLED=1 - - set GORACE=history_size=7 - - copy "%TASK_USER_CREDENTIALS%" "%CD%\next-task-user.json" - # We must set here since this worker does not have a Z: drive - - set GW_SKIP_Z_DRIVE_TESTS=true - - 'go test -tags multiuser -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v -race ./...' - - set GW_TESTS_RUN_AS_CURRENT_USER=true - - 'go test -tags multiuser -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v -race' - - ineffassign . - artifacts: - - name: public/build/generic-worker-windows-amd64.exe - path: gopath1.10.8\bin\generic-worker.exe - expires: "{{ '2 weeks' | $fromNow }}" - type: file - mounts: - - cacheName: generic-worker-checkout - directory: gopath1.10.8\src - - content: - url: https://storage.googleapis.com/golang/go1.10.8.windows-amd64.zip - sha256: ab63b55c349f75cce4b93aefa9b52828f50ebafb302da5057db0e686d7873d7a - directory: go1.10.8 - format: zip - - content: - url: https://github.com/git-for-windows/git/releases/download/v2.14.1.windows.1/MinGit-2.14.1-64-bit.zip - sha256: 65c12e4959b8874187b68ec37e532fe7fc526e10f6f0f29e699fa1d2449e7d92 - directory: git - format: zip - - - ########################################################## - ############### Windows 7 Multiuser Build ################ - ########################################################## - - - provisionerId: aws-provisioner-v1 - workerType: gecko-t-win7-32-cu - metadata: - name: "Build/test 32 bit generic-worker (multiuser engine) on Windows 7" - description: "This builds and tests the 32 bit Windows version of generic-worker (multiuser engine) on Windows 7" - owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here - source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here - extra: - github: - # Events that will trigger this task - events: - - pull_request.opened - - pull_request.synchronize - - push - scopes: - - generic-worker:cache:generic-worker-checkout - - secrets:get:repo:github.com/taskcluster/generic-worker - payload: - features: - taskclusterProxy: true - maxRunTime: 3600 - env: - # Python 2 is needed for TestDesktopResizeAndMovePointer - # Powershell is needed for TestMounts - # System32 is needed for e.g. cmd.exe, icacls.exe, ... - PATH: C:\Windows\System32;C:\Windows\System32\WindowsPowerShell\v1.0;C:\mozilla-build\python - command: - - >- - C:\mozilla-build\wget\wget.exe -q -O- - %TASKCLUSTER_PROXY_URL%/secrets/v1/secret/repo:github.com/taskcluster/generic-worker - | C:\mozilla-build\msys\bin\sed.exe -n - 's/.*"b64_encoded_credentials_batch_script": "\(.*\)".*/\1/p' > tc-creds.bat.b64 - - certutil -decode tc-creds.bat.b64 tc-creds.bat - - call tc-creds.bat 2>&1 - - set CGO_ENABLED=0 - - set GOPATH=%CD%\gopath1.10.8 - - set GOROOT=%CD%\go1.10.8\go - - set PATH=%CD%\git\bin;%GOPATH%\bin;%GOROOT%\bin;%PATH% - - git config --global core.autocrlf false - - go version - - go env - - 'if not exist "%GOPATH%\src\github.com\taskcluster" mkdir "%GOPATH%\src\github.com\taskcluster"' - - 'cd "%GOPATH%\src\github.com\taskcluster"' - - 'if not exist generic-worker git clone {{ event.head.repo.url }} generic-worker' - - 'cd generic-worker' - - 'git fetch "{{ event.head.repo.url }}" "+{{ event.head.ref }}:refs/heads/X%TASK_ID%"' - - 'git checkout -f "X%TASK_ID%"' - - 'git reset --hard "{{ event.head.sha }}"' - - 'git clean -fdx' - - 'git checkout -B tmp -t "X%TASK_ID%"' - - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - - cd gw-codegen - - go get -v - - cd .. - - go generate - - set revision={{ event.head.sha }} - - go install -tags multiuser -v -ldflags "-X main.revision=%revision%" ./... - - set GORACE=history_size=7 - - copy "%TASK_USER_CREDENTIALS%" "%CD%\next-task-user.json" - # We must set here since this worker type runs tasks from Z: drive - - set GW_SKIP_Z_DRIVE_TESTS=true - - 'go test -tags multiuser -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v ./...' - - set GW_TESTS_RUN_AS_CURRENT_USER=true - - 'go test -tags multiuser -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v' - - ineffassign . - artifacts: - - name: public/build/generic-worker-windows-386.exe - path: gopath1.10.8\bin\generic-worker.exe - expires: "{{ '2 weeks' | $fromNow }}" - type: file - mounts: - - cacheName: generic-worker-checkout - directory: gopath1.10.8\src - - content: - url: https://storage.googleapis.com/golang/go1.10.8.windows-386.zip - sha256: 9ded97d830bef3734ea6de70df0159656d6a63e01484175b34d72b8db326bda0 - directory: go1.10.8 - format: zip - - content: - url: https://github.com/git-for-windows/git/releases/download/v2.11.0.windows.3/Git-2.11.0.3-32-bit.tar.bz2 - sha256: 0f0e2f78fc9b91d6c860eb7de742f3601b0ccd13c5c61444c7cf55b00bcb4ed4 - directory: git - format: tar.bz2 - - - ########################################################## - ############### Windows 10 Multiuser Build ############### - ########################################################## - - - provisionerId: aws-provisioner-v1 - workerType: gecko-t-win10-64-cu - metadata: - name: "Build/test 64 bit generic-worker (multiuser engine) on Windows 10" - description: "This builds and tests the 64 bit Windows version of generic-worker (multiuser engine) on Windows 10" - owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here - source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here - extra: - github: - # Events that will trigger this task - events: - - pull_request.opened - - pull_request.synchronize - - push - scopes: - - generic-worker:cache:generic-worker-checkout - - secrets:get:repo:github.com/taskcluster/generic-worker - payload: - features: - taskclusterProxy: true - maxRunTime: 3600 - command: - - >- - C:\mozilla-build\bin\wget.exe -q -O- - %TASKCLUSTER_PROXY_URL%/secrets/v1/secret/repo:github.com/taskcluster/generic-worker - | C:\mozilla-build\msys\bin\sed.exe -n - 's/.*"b64_encoded_credentials_batch_script": "\(.*\)".*/\1/p' > tc-creds.bat.b64 - - certutil -decode tc-creds.bat.b64 tc-creds.bat - - call tc-creds.bat 2>&1 - - set CGO_ENABLED=0 - - set GOPATH=%CD%\gopath1.10.8 - - set GOROOT=%CD%\go1.10.8\go - - set PATH=%CD%\git\cmd;%GOPATH%\bin;%GOROOT%\bin;%PATH% - - git config --global core.autocrlf false - - go version - - go env - - 'if not exist "%GOPATH%\src\github.com\taskcluster" mkdir "%GOPATH%\src\github.com\taskcluster"' - - 'cd "%GOPATH%\src\github.com\taskcluster"' - - 'if not exist generic-worker git clone {{ event.head.repo.url }} generic-worker' - - 'cd generic-worker' - - 'git fetch "{{ event.head.repo.url }}" "+{{ event.head.ref }}:refs/heads/X%TASK_ID%"' - - 'git checkout -f "X%TASK_ID%"' - - 'git reset --hard "{{ event.head.sha }}"' - - 'git clean -fdx' - - 'git checkout -B tmp -t "X%TASK_ID%"' - - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - - cd gw-codegen - - go get -v - - cd .. - - go generate - - | - :: this counts the number of lines returned by git status - :: dump temp file a directory higher, otherwise git status reports the tmp1.txt file! - git status --porcelain | C:\Windows\System32\find.exe /v /c "" > ..\tmp1.txt - set /P lines=<..\tmp1.txt - :: this checks that if more than 0 lines are returned, we fail - if %lines% gtr 0 exit /b 64 - :: find.exe will have exited with exit code 1, so need to explicitly exit with 0 - exit /b 0 - - git rev-parse HEAD > revision.txt - - set /p revision=< revision.txt - - go install -tags multiuser -v -ldflags "-X main.revision=%revision%" ./... - - set CGO_ENABLED=1 - - set GORACE=history_size=7 - - copy "%TASK_USER_CREDENTIALS%" "%CD%\next-task-user.json" - # We must set here since this worker type runs tasks from Z: drive - - set GW_SKIP_Z_DRIVE_TESTS=true - - 'go test -tags multiuser -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v -race ./...' - - set GW_TESTS_RUN_AS_CURRENT_USER=true - - 'go test -tags multiuser -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=%revision%" -v -race' - - ineffassign . - artifacts: - - name: public/build/generic-worker-windows-amd64.exe - path: gopath1.10.8\bin\generic-worker.exe - expires: "{{ '2 weeks' | $fromNow }}" - type: file - mounts: - - cacheName: generic-worker-checkout - directory: gopath1.10.8\src - - content: - url: https://storage.googleapis.com/golang/go1.10.8.windows-amd64.zip - sha256: ab63b55c349f75cce4b93aefa9b52828f50ebafb302da5057db0e686d7873d7a - directory: go1.10.8 - format: zip - - content: - url: https://github.com/git-for-windows/git/releases/download/v2.14.1.windows.1/MinGit-2.14.1-64-bit.zip - sha256: 65c12e4959b8874187b68ec37e532fe7fc526e10f6f0f29e699fa1d2449e7d92 - directory: git - format: zip - - - ########################################################## - ############ macOS Mojave 10.14 Simple Build ############# - ########################################################## - - - provisionerId: pmoore-manual - workerType: mac-os-x - metadata: - name: "Build/test 64 bit generic-worker (simple engine) on macOS Mojave 10.14" - description: "This builds the 64 bit macOS version of generic-worker (simple engine)" - owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here - source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here - extra: - github: - # Events that will trigger this task - events: - - pull_request.opened - - pull_request.synchronize - - push - scopes: - - generic-worker:cache:generic-worker-checkout - - secrets:get:repo:github.com/taskcluster/generic-worker - payload: - features: - taskclusterProxy: true - maxRunTime: 3600 - command: - - - /bin/bash - - -vxec - - | - export CGO_ENABLED=0 - export GOROOT="$(pwd)/go1.10.8/go" - export GOPATH="$(pwd)/gopath1.10.8" - export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" - go version - go env - curl -s "${TASKCLUSTER_PROXY_URL}/secrets/v1/secret/repo:github.com/taskcluster/generic-worker" | sed -n 's/.*"b64_encoded_credentials_script": "\(.*\)".*/\1/p' | base64 -D > tc-creds.sh - source tc-creds.sh - mkdir -p "${GOPATH}/src/github.com/taskcluster" - cd "${GOPATH}/src/github.com/taskcluster" - if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi - cd 'generic-worker' - git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" - git checkout -f "X${TASK_ID}" - git reset --hard '{{ event.head.sha }}' - git clean -fdx - git checkout -B tmp -t "X${TASK_ID}" - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - cd gw-codegen - go get -v - cd .. - go generate - go install -tags simple -v -ldflags "-X main.revision=$(git rev-parse HEAD)" ./... - # output of wc command can contain spaces on darwin, so no quotes around expression - test $(git status --porcelain | wc -l) == 0 - GORACE=history_size=7 CGO_ENABLED=1 go test -tags simple -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... - ineffassign . - artifacts: - - name: public/build/generic-worker-darwin-amd64 - path: gopath1.10.8/bin/generic-worker - expires: "{{ '2 weeks' | $fromNow }}" - type: file - mounts: -# - cacheName: generic-worker-checkout -# directory: gopath1.10.8/src - - content: - url: https://storage.googleapis.com/golang/go1.10.8.darwin-amd64.tar.gz - sha256: f41bc914a721ac98a187df824b3b40f0a7f35bfb3c6d31221bdd940d537d3c28 - directory: go1.10.8 - format: tar.gz - - - ########################################################## - ########### macOS Mojave 10.14 Multiuser Build ########### - ########################################################## - - - provisionerId: pmoore-manual - workerType: mac-os-x - metadata: - name: "Build/test 64 bit generic-worker (multiuser engine) on macOS Mojave 10.14" - description: "This builds the 64 bit macOS version of generic-worker (multiuser engine)" - owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here - source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here - extra: - github: - # Events that will trigger this task - events: - - pull_request.opened - - pull_request.synchronize - - push - scopes: - - generic-worker:cache:generic-worker-checkout - - secrets:get:repo:github.com/taskcluster/generic-worker - payload: - features: - taskclusterProxy: true - maxRunTime: 3600 - env: - command: - - - /bin/bash - - -vxec - - | - export CGO_ENABLED=0 - export GOROOT="$(pwd)/go1.10.8/go" - export GOPATH="$(pwd)/gopath1.10.8" - export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" - go version - go env - curl -s "${TASKCLUSTER_PROXY_URL}/secrets/v1/secret/repo:github.com/taskcluster/generic-worker" | sed -n 's/.*"b64_encoded_credentials_script": "\(.*\)".*/\1/p' | base64 -D > tc-creds.sh - source tc-creds.sh - mkdir -p "${GOPATH}/src/github.com/taskcluster" - cd "${GOPATH}/src/github.com/taskcluster" - if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi - cd 'generic-worker' - git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" - git checkout -f "X${TASK_ID}" - git reset --hard '{{ event.head.sha }}' - git clean -fdx - git checkout -B tmp -t "X${TASK_ID}" - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - cd gw-codegen - go get -v - cd .. - go generate - go install -tags multiuser -v -ldflags "-X main.revision=$(git rev-parse HEAD)" ./... - # output of wc command can contain spaces on darwin, so no quotes around expression - test $(git status --porcelain | wc -l) == 0 - cp "${TASK_USER_CREDENTIALS}" next-task-user.json - # IMPORTANT - run go test with GW_TESTS_RUN_AS_CURRENT_USER=true *before* running it without - # otherwise tests that call `go run ....` will write go object files to .cache as root - GW_TESTS_RUN_AS_CURRENT_USER=true GORACE=history_size=7 CGO_ENABLED=1 go test -tags multiuser -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v -race - GORACE=history_size=7 CGO_ENABLED=1 go test -tags multiuser -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... - ineffassign . - artifacts: - - name: public/build/generic-worker-darwin-amd64 - path: gopath1.10.8/bin/generic-worker - expires: "{{ '2 weeks' | $fromNow }}" - type: file - mounts: -# - cacheName: generic-worker-checkout -# directory: gopath1.10.8/src - - content: - url: https://storage.googleapis.com/golang/go1.10.8.darwin-amd64.tar.gz - sha256: f41bc914a721ac98a187df824b3b40f0a7f35bfb3c6d31221bdd940d537d3c28 - directory: go1.10.8 - format: tar.gz - - - ########################################################## - ################ Linux ARM6 Simple Build ################# - ########################################################## - - - ########################################################## - # Currently Raspberry PI is not available (being used for other things) - # Will reenable when I have it back again - ########################################################## - - # - provisionerId: pmoore-manual - # workerType: raspberry-pi-3b - # metadata: - # name: "Build/test ARM6 generic-worker (simple engine) on Linux (Raspberry Pi)" - # description: "This builds the ARM6 Linux version of generic-worker (simple engine)" - # owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here - # source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here - # extra: - # github: - # # Events that will trigger this task - # events: - # - pull_request.opened - # - pull_request.synchronize - # - push - # scopes: - # - generic-worker:cache:generic-worker-checkout - # - secrets:get:repo:github.com/taskcluster/generic-worker - # payload: - # features: - # taskclusterProxy: true - # maxRunTime: 3600 - # command: - # - - /bin/bash - # - -vxec - # - | - # export CGO_ENABLED=0 - # export GOROOT="$(pwd)/go1.10.8/go" - # export GOPATH="$(pwd)/gopath1.10.8" - # export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" - # export CGO_ENABLED=0 - # go version - # go env - # curl -s "${TASKCLUSTER_PROXY_URL}/secrets/v1/secret/repo:github.com/taskcluster/generic-worker" | sed -n 's/.*"b64_encoded_credentials_script": "\(.*\)".*/\1/p' | base64 -d > tc-creds.sh - # source tc-creds.sh - # mkdir -p "${GOPATH}/src/github.com/taskcluster" - # cd "${GOPATH}/src/github.com/taskcluster" - # if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi - # cd 'generic-worker' - # git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" - # git checkout -f "X${TASK_ID}" - # git reset --hard '{{ event.head.sha }}' - # git clean -fdx - # git checkout -B tmp -t "X${TASK_ID}" - # go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - # cd gw-codegen - # go get -v - # cd .. - # go generate - # go install -tags simple -v -ldflags "-X main.revision=$(git rev-parse HEAD)" ./... - # test "$(git status --porcelain | wc -l)" == 0 - # GORACE=history_size=7 go test -tags simple -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v ./... - # ineffassign . - # artifacts: - # - name: public/build/generic-worker-linux-armv6l - # path: gopath1.10.8/bin/generic-worker - # expires: "{{ '2 weeks' | $fromNow }}" - # type: file - # mounts: - ## - cacheName: generic-worker-checkout - ## directory: gopath1.10.8/src - # - content: - # url: https://storage.googleapis.com/golang/go1.10.8.linux-armv6l.tar.gz - # sha256: 6fdbc67524fc4c15fc87014869dddce9ecda7958b78f3cb1bbc5b0a9b61bfb95 - # directory: go1.10.8 - # format: tar.gz - - - ########################################################## - ############## Linux amd64 Multiuser Build ############### - ########################################################## - - - provisionerId: aws-provisioner-v1 - workerType: gwci-linux - metadata: - name: "Build/test 64 bit generic-worker (multiuser engine) on Ubuntu 17.04 VM" - description: "This builds the 64 bit linux version of generic-worker (multiuser engine)" - owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here - source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here - extra: - github: - # Events that will trigger this task - events: - - pull_request.opened - - pull_request.synchronize - - push - scopes: - - generic-worker:cache:generic-worker-checkout - - secrets:get:repo:github.com/taskcluster/generic-worker - payload: - features: - taskclusterProxy: - true - maxRunTime: 3600 - command: - - - /bin/bash - - -vxec - - | - export CGO_ENABLED=0 - export GOROOT="$(pwd)/go1.10.8/go" - export GOPATH="$(pwd)/gopath1.10.8" - export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" - go version - go env - curl -s "${TASKCLUSTER_PROXY_URL}/secrets/v1/secret/repo:github.com/taskcluster/generic-worker" | sed -n 's/.*"b64_encoded_credentials_script": "\(.*\)".*/\1/p' | base64 -d > tc-creds.sh - source tc-creds.sh - mkdir -p "${GOPATH}/src/github.com/taskcluster" - cd "${GOPATH}/src/github.com/taskcluster" - if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi - cd 'generic-worker' - git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" - git checkout -f "X${TASK_ID}" - git reset --hard '{{ event.head.sha }}' - git clean -fdx - git checkout -B tmp -t "X${TASK_ID}" - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - cd gw-codegen - go get -v - cd .. - go generate - go install -tags multiuser -v -ldflags "-X main.revision=$(git rev-parse HEAD)" ./... - test "$(git status --porcelain | wc -l)" == 0 - cp "${TASK_USER_CREDENTIALS}" next-task-user.json - # IMPORTANT - run go test with GW_TESTS_RUN_AS_CURRENT_USER=true *before* running it without - # otherwise tests that call `go run ....` will write go object files to .cache as root - GW_TESTS_RUN_AS_CURRENT_USER=true GORACE=history_size=7 CGO_ENABLED=1 go test -tags multiuser -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v -race - GORACE=history_size=7 CGO_ENABLED=1 go test -tags multiuser -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... - "${GOPATH}/bin/ineffassign" . - artifacts: - - name: public/build/generic-worker-linux-amd64 - path: gopath1.10.8/bin/generic-worker - expires: "{{ '2 weeks' | $fromNow }}" - type: file - mounts: -# - cacheName: generic-worker-checkout -# directory: gopath1.10.8/src - - content: - url: https://storage.googleapis.com/golang/go1.10.8.linux-amd64.tar.gz - sha256: d8626fb6f9a3ab397d88c483b576be41fa81eefcec2fd18562c87626dbb3c39e - directory: go1.10.8 - format: tar.gz - - - ########################################################## - ################ Linux amd64 Simple Build ################ - ########################################################## - - - provisionerId: aws-provisioner-v1 - workerType: gwci-linux - metadata: - name: "Build/test 64 bit generic-worker (simple engine) on Ubuntu 17.04 VM" - description: "This builds the 64 bit linux version of generic-worker (simple engine)" - owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here - source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here - extra: - github: - # Events that will trigger this task - events: - - pull_request.opened - - pull_request.synchronize - - push - scopes: - - generic-worker:cache:generic-worker-checkout - - secrets:get:repo:github.com/taskcluster/generic-worker - payload: - features: - taskclusterProxy: - true - maxRunTime: 3600 - command: - - - /bin/bash - - -vxec - - | - export CGO_ENABLED=0 - export GOROOT="$(pwd)/go1.10.8/go" - export GOPATH="$(pwd)/gopath1.10.8" - export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" - go version - go env - wget -q -O- "${TASKCLUSTER_PROXY_URL}/secrets/v1/secret/repo:github.com/taskcluster/generic-worker" | sed -n 's/.*"b64_encoded_credentials_script": "\(.*\)".*/\1/p' | base64 -d > tc-creds.sh - source tc-creds.sh - mkdir -p "${GOPATH}/src/github.com/taskcluster" - cd "${GOPATH}/src/github.com/taskcluster" - if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi - cd 'generic-worker' - git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" - git checkout -f "X${TASK_ID}" - git reset --hard '{{ event.head.sha }}' - git clean -fdx - git checkout -B tmp -t "X${TASK_ID}" - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - cd gw-codegen - go get -v - cd .. - go generate - go install -tags simple -v -ldflags "-X main.revision=$(git rev-parse HEAD)" ./... - test "$(git status --porcelain | wc -l)" == 0 - GORACE=history_size=7 CGO_ENABLED=1 go test -tags simple -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... - "${GOPATH}/bin/ineffassign" . - artifacts: - - name: public/build/generic-worker-linux-amd64 - path: gopath1.10.8/bin/generic-worker - expires: "{{ '2 weeks' | $fromNow }}" - type: file - mounts: -# - cacheName: generic-worker-checkout -# directory: gopath1.10.8/src - - content: - url: https://storage.googleapis.com/golang/go1.10.8.linux-amd64.tar.gz - sha256: d8626fb6f9a3ab397d88c483b576be41fa81eefcec2fd18562c87626dbb3c39e - directory: go1.10.8 - format: tar.gz - - - ########################################################## - ################ Linux amd64 Docker Build ################ - ########################################################## - - - provisionerId: aws-provisioner-v1 - workerType: gwci-linux - metadata: - name: "Build/test 64 bit generic-worker (docker engine) on Ubuntu 17.04 VM" - description: "This builds the 64 bit linux version of generic-worker (docker engine)" - owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here - source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here - extra: - github: - # Events that will trigger this task - events: - - pull_request.opened - - pull_request.synchronize - - push - scopes: - - generic-worker:cache:generic-worker-checkout - - secrets:get:repo:github.com/taskcluster/generic-worker - payload: - features: - taskclusterProxy: - true - maxRunTime: 3600 - command: - - - /bin/bash - - -vxec - - | - export CGO_ENABLED=0 - export GOROOT="$(pwd)/go1.10.8/go" - export GOPATH="$(pwd)/gopath1.10.8" - export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" - go version - go env - wget -q -O- "${TASKCLUSTER_PROXY_URL}/secrets/v1/secret/repo:github.com/taskcluster/generic-worker" | sed -n 's/.*"b64_encoded_credentials_script": "\(.*\)".*/\1/p' | base64 -d > tc-creds.sh - source tc-creds.sh - mkdir -p "${GOPATH}/src/github.com/taskcluster" - cd "${GOPATH}/src/github.com/taskcluster" - if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi - cd 'generic-worker' - git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" - git checkout -f "X${TASK_ID}" - git reset --hard '{{ event.head.sha }}' - git clean -fdx - git checkout -B tmp -t "X${TASK_ID}" - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign - cd gw-codegen - go get -v - cd .. - go generate - go install -tags docker -v -ldflags "-X main.revision=$(git rev-parse HEAD)" ./... - test "$(git status --porcelain | wc -l)" == 0 - GORACE=history_size=7 CGO_ENABLED=1 go test -tags docker -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=$(git rev-parse HEAD)" -v -race ./... - "${GOPATH}/bin/ineffassign" . - artifacts: - - name: public/build/generic-worker-linux-amd64 - path: gopath1.10.8/bin/generic-worker - expires: "{{ '2 weeks' | $fromNow }}" - type: file - mounts: -# - cacheName: generic-worker-checkout -# directory: gopath1.10.8/src - - content: - url: https://storage.googleapis.com/golang/go1.10.8.linux-amd64.tar.gz - sha256: d8626fb6f9a3ab397d88c483b576be41fa81eefcec2fd18562c87626dbb3c39e - directory: go1.10.8 - format: tar.gz - - - ########################################################## - ############## Check vendored dependencies ############### - ########################################################## - - - provisionerId: aws-provisioner-v1 - workerType: gwci-linux - metadata: - name: "Run 'dep check' to ensure dependencies are up to date" - description: "Run 'dep check' to ensure dependencies are up to date" - owner: "{{ event.head.user.email }}" # the user who sent the pr/push e-mail will be inserted here - source: "{{ event.head.repo.url }}" # the repo where the pr came from will be inserted here - extra: - github: - # Events that will trigger this task - events: - - pull_request.opened - - pull_request.synchronize - - push - scopes: - - generic-worker:cache:generic-worker-checkout - payload: - maxRunTime: 3600 - command: - - - /bin/bash - - -vxec - - | - export CGO_ENABLED=0 - export GOROOT="$(pwd)/go1.10.8/go" - export GOPATH="$(pwd)/gopath1.10.8" - export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" - go version - go env - mkdir -p "${GOPATH}/src/github.com/taskcluster" - cd "${GOPATH}/src/github.com/taskcluster" - if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone '{{ event.head.repo.url }}' 'generic-worker'; fi - cd 'generic-worker' - git fetch '{{ event.head.repo.url }}' "+{{ event.head.ref }}:refs/heads/X${TASK_ID}" - git checkout -f "X${TASK_ID}" - git reset --hard '{{ event.head.sha }}' - git clean -fdx - git checkout -B tmp -t "X${TASK_ID}" - mkdir -p "${GOPATH}/bin" - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep check - mounts: -# - cacheName: generic-worker-checkout -# directory: gopath1.10.8/src - - content: - url: https://storage.googleapis.com/golang/go1.10.8.linux-amd64.tar.gz - sha256: d8626fb6f9a3ab397d88c483b576be41fa81eefcec2fd18562c87626dbb3c39e - directory: go1.10.8 - format: tar.gz + $let: + should_run: + $if: 'tasks_for == "github-pull-request"' + then: {$eval: 'event["action"] in ["opened", "reopened", "synchronize"]'} + else: {$eval: 'tasks_for == "github-push"'} + clone_url: + $if: 'tasks_for == "github-pull-request"' + then: ${event.pull_request.head.repo.clone_url} + else: ${event.repository.clone_url} + sha: + $if: 'tasks_for == "github-pull-request"' + then: ${event.pull_request.head.sha} + else: ${event.after} + in: + - $if: should_run + then: + taskId: {$eval: as_slugid("decision")} + created: {$fromNow: ''} + deadline: {$fromNow: '2 hours'} + provisionerId: proj-taskcluster + workerType: gw-ci-ubuntu-18-04 + payload: + maxRunTime: 3600 + env: + GITHUB_SHA: '${sha}' + GITHUB_CLONE_URL: '${clone_url}' + command: + - - /bin/bash + - -vxec + - | + export CGO_ENABLED=0 + export GOROOT="$(pwd)/go1.10.8/go" + export GOPATH="$(pwd)/gopath1.10.8" + export PATH="$${GOPATH}/bin:$${GOROOT}/bin:$${PATH}" + go version + go env + mkdir -p "$${GOPATH}/src/github.com/taskcluster" + cd "$${GOPATH}/src/github.com/taskcluster" + if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone "$${GITHUB_CLONE_URL}" 'generic-worker'; fi + cd 'generic-worker' + git fetch "$${GITHUB_CLONE_URL}" "+$${GITHUB_SHA}:refs/heads/X$${TASK_ID}" + git checkout -f "X$${TASK_ID}" + git reset --hard "$${GITHUB_SHA}" + git clean -fdx + git checkout -B tmp -t "X$${TASK_ID}" + mkdir -p "$${GOPATH}/bin" + cd gw-decision-task + go install + gw-decision-task tasks.yml "$${GITHUB_SHA}" + features: + taskclusterProxy: true + mounts: + # - cacheName: generic-worker-checkout + # directory: gopath1.10.8/src + - content: + url: https://storage.googleapis.com/golang/go1.10.8.linux-amd64.tar.gz + sha256: d8626fb6f9a3ab397d88c483b576be41fa81eefcec2fd18562c87626dbb3c39e + directory: go1.10.8 + format: tar.gz + metadata: + name: Generic Worker CI Decision Task + description: Generates the tasks that build and test generic-worker + owner: taskcluster-internal@mozilla.com + source: ${clone_url} + scopes: + - generic-worker:cache:generic-worker-checkout + - secrets:get:project/taskcluster/testing/generic-worker/ci-creds + - queue:scheduler-id:taskcluster-github + - queue:create-task:highest:proj-taskcluster/gw-ci-* diff --git a/Gopkg.lock b/Gopkg.lock index 8ac78d3b..416817fc 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -318,11 +318,11 @@ [[projects]] branch = "master" - digest = "1:951106573999407dd9676fcdf4c6184b58fc58055be3e1af54f109a451dda786" + digest = "1:d0e9402943b249967ff519c601b944901bd757bf766cc0687399ac3afac2f758" name = "github.com/taskcluster/shell" packages = ["."] pruneopts = "UT" - revision = "70c77c91e399f7116e6aa2c6e7abc18946ec4ada" + revision = "c688067f12d36940aeeb557b74e3838b30beb427" [[projects]] branch = "master" @@ -517,6 +517,7 @@ "github.com/gorilla/websocket", "github.com/kr/text", "github.com/mholt/archiver", + "github.com/pborman/uuid", "github.com/peterbourgon/mergemap", "github.com/sirupsen/logrus", "github.com/stretchr/testify/assert", diff --git a/README.md b/README.md index 1538ca82..f44d9294 100644 --- a/README.md +++ b/README.md @@ -794,6 +794,16 @@ go test -v ./... There are a few environment variables that you can set to influence the tests: +### `GW_SKIP_PYTHON_TESTS` + +Set to a non-empty string if you wish to skip all tests that require python to +be installed. + +### `GW_SKIP_MOZILLA_BUILD_TESTS` + +Set to a non-empty string if you wish to skip all tests that require +mozilla-build to be installed. + ### `GW_SKIP_INTEGRATION_TESTS` Set to a non-empty string if you wish to skip all tests that submit tasks to a diff --git a/aws_helper_test.go b/aws_helper_test.go index 82eb849c..afd89928 100644 --- a/aws_helper_test.go +++ b/aws_helper_test.go @@ -213,7 +213,7 @@ func (m *MockAWSProvisionedEnvironment) Setup(t *testing.T) (teardown func(), er "availabilityZone": "outer-space", "privateIp": "87.65.43.21", "version": "2017-09-30", - "instanceId": "test-instance-id", + "instanceId": "test-worker-id", "instanceType": "p3.teenyweeny", "accountId": "123456789012", "imageId": "test-ami", diff --git a/gw-decision-task/main.go b/gw-decision-task/main.go new file mode 100644 index 00000000..6ee98f32 --- /dev/null +++ b/gw-decision-task/main.go @@ -0,0 +1,324 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/ghodss/yaml" + "github.com/taskcluster/shell" + "github.com/taskcluster/slugid-go/slugid" + tcclient "github.com/taskcluster/taskcluster-client-go" + "github.com/taskcluster/taskcluster-client-go/tcqueue" +) + +// Data types that map to sections of tasks.yml +type ( + TasksConfig struct { + Types map[string]*Type `json:"Types"` + Tasks map[string][]*Task `json:"Tasks"` + WorkerPools map[string]*WorkerPool `json:"WorkerPools"` + Commands map[string]*CommandSet `json:"Commands"` + Mounts map[string]*Mount `json:"Mounts"` + } + Type struct { + Name string `json:"Name"` + Description string `json:"Description"` + Mounts []string `json:"Mounts"` + Command string `json:"Command"` + Features map[string]interface{} `json:"Features"` + Scopes []string `json:"Scopes"` + Artifacts []*Artifact `json:"Artifacts"` + MaxRunTime uint `json:"MaxRunTime"` + } + Artifact struct { + Name string `json:"name"` + Path string `json:"path"` + Type string `json:"type"` + } + Task struct { + WorkerPool string `json:"WorkerPool"` + Env map[string]string `json:"Env"` + } + WorkerPool struct { + Platform string `json:"Platform"` + OS string `json:"OS"` + Arch string `json:"Arch"` + } + CommandSet struct { + Posix [][]string `json:"Posix"` + Windows []string `json:"Windows"` + } + Mount struct { + Directory string `json:"directory"` + File string `json:"file"` + Content map[string]map[string]*Content `json:"content"` + } + Content struct { + URL string `json:"url"` + SHA256 string `json:"sha256"` + Format string `json:"format"` + } +) + +// Internal types used by program +type ( + DecisionTask struct { + TasksConfig *TasksConfig + TaskID string + TaskGroupID string + } + TaskGroup struct { + // map from taskID to task definition + taskDefs map[string]*tcqueue.TaskDefinitionRequest + } +) + +func main() { + if len(os.Args) != 3 { + log.Printf("Usage: %v TASKS_YAML_FILE GIT_REVISION", os.Args[0]) + log.Fatalf("You ran: %v", shell.Escape(os.Args...)) + } + decisionTask, err := NewDecisionTask(os.Args[1], os.Args[2]) + if err != nil { + log.Fatalf("%s", err) + } + err = decisionTask.Execute() + if err != nil { + log.Fatalf("%s", err) + } +} + +func NewDecisionTask(yamlPath string, gitRevision string) (*DecisionTask, error) { + absYAMLPath, err := filepath.Abs(yamlPath) + if err != nil { + return nil, fmt.Errorf("Could not determine absolute file location of decision task YAML config file %q: %s", yamlPath, err) + } + data, err := ioutil.ReadFile(absYAMLPath) + if err != nil { + return nil, fmt.Errorf("Could not read decision task YAML config file %q: %s", absYAMLPath, err) + } + // JSON is valid YAML, so we can safely convert, even if it is already JSON + rawJSON, err := yaml.YAMLToJSON(data) + if err != nil { + return nil, fmt.Errorf("Could not interpret decision task YAML config file %q as YAML: %s", absYAMLPath, err) + } + tc := new(TasksConfig) + dec := json.NewDecoder(bytes.NewBuffer(rawJSON)) + dec.DisallowUnknownFields() + err = dec.Decode(tc) + if err != nil { + return nil, fmt.Errorf("Decision task YAML config file %q has invalid content: %s", absYAMLPath, err) + } + + d := &DecisionTask{ + TasksConfig: tc, + } + // TaskID will be "" if running outside of taskcluster, e.g. locally by a developer + d.TaskID = os.Getenv("TASK_ID") + if d.TaskID != "" { + d.TaskGroupID = d.TaskID + } else { + // If running decision task code outside of taskcluster (e.g. developer + // manually runs `gw-decision-task tasks.yml`), then still generate + // tasks, but do not make them dependent on the decision task, since + // there isn't one. However, still place all generated tasks in the + // same task group, so create a new taskGroupId. + d.TaskGroupID = slugid.Nice() + } + return d, nil +} + +func (dt *DecisionTask) Execute() (err error) { + tasks, err := dt.GenerateTasks() + if err != nil { + return err + } + return tasks.Submit() +} + +func (dt *DecisionTask) GenerateTasks() (*TaskGroup, error) { + tg := &TaskGroup{ + taskDefs: map[string]*tcqueue.TaskDefinitionRequest{}, + } + for taskType, tasks := range dt.TasksConfig.Tasks { + for _, task := range tasks { + + typ := dt.TasksConfig.Types[taskType] + + workerPool := dt.TasksConfig.WorkerPools[task.WorkerPool] + commandSet := dt.TasksConfig.Commands[typ.Command] + + // context contains all variable names that can be referred to in + // task name/description, artifact name/path in ${VARNAME} format + context := map[string]string{ + "PLATFORM": workerPool.Platform, + "OS": workerPool.OS, + "ARCH": workerPool.Arch, + } + // add task env vars to context, so that task name/description + // can refer to them + for k, v := range task.Env { + context[k] = v + } + var command interface{} + if workerPool.OS == "windows" { + command = commandSet.Windows + context["EXTENSION"] = ".exe" + } else { + command = commandSet.Posix + context["EXTENSION"] = "" + } + + taskName := substituteVars(context, typ.Name) + taskDescription := substituteVars(context, typ.Description) + + mounts := []map[string]interface{}{} + + for _, mountName := range typ.Mounts { + mount := dt.TasksConfig.Mounts[mountName] + osContent := mount.Content[workerPool.OS] + if osContent == nil { + osContent = mount.Content["all"] + } + content := osContent[workerPool.Arch] + if content == nil { + content = osContent["all"] + } + if content != nil { + mountEntry := map[string]interface{}{ + "content": map[string]string{ + "url": content.URL, + "sha256": content.SHA256, + }, + } + if mount.Directory != "" { + mountEntry["directory"] = mount.Directory + } + if mount.File != "" { + mountEntry["file"] = mount.File + } + if content.Format != "" { + mountEntry["format"] = content.Format + } + mounts = append(mounts, mountEntry) + } + } + + payload := map[string]interface{}{ + "env": map[string]string{ + "GITHUB_SHA": os.Getenv("GITHUB_SHA"), + "GITHUB_CLONE_URL": os.Getenv("GITHUB_CLONE_URL"), + }, + } + for k, v := range task.Env { + payload["env"].(map[string]string)[k] = v + } + if typ.MaxRunTime > 0 { + payload["maxRunTime"] = typ.MaxRunTime + } + artifacts := make([]*Artifact, len(typ.Artifacts), len(typ.Artifacts)) + for i, a := range typ.Artifacts { + artifacts[i] = &Artifact{ + Name: substituteVars(context, a.Name), + Path: substituteVars(context, a.Path), + Type: a.Type, + } + } + for key, value := range map[string]interface{}{ + "artifacts": artifacts, + "command": command, + "features": typ.Features, + "mounts": mounts, + } { + if reflect.ValueOf(value).IsValid() { + if !reflect.ValueOf(value).IsNil() { + payload[key] = value + } + } + } + scopes := typ.Scopes + td, err := dt.TaskDefinition(task.WorkerPool, taskName, taskDescription, scopes, payload) + if err != nil { + return nil, err + } + tg.taskDefs[slugid.Nice()] = td + } + } + return tg, nil +} + +func substituteVars(context map[string]string, expression string) string { + result := expression + for k, v := range context { + result = strings.Replace(result, "${"+k+"}", v, -1) + } + return result +} + +func (dt *DecisionTask) TaskDefinition(workerPool string, name string, description string, scopes []string, payload interface{}) (*tcqueue.TaskDefinitionRequest, error) { + workerPoolSplit := strings.Split(workerPool, "/") + if len(workerPoolSplit) != 2 { + return nil, fmt.Errorf("Worker pool %q should contain precisely one '/' but contains %v", workerPool, len(workerPoolSplit)) + } + provisionerID := workerPoolSplit[0] + workerType := workerPoolSplit[1] + + var dependencies []string + var schedulerID string + // Are we running inside a task, or being run e.g. locally by a developer? + if dt.TaskID != "" { + dependencies = []string{dt.TaskID} + schedulerID = "taskcluster-github" + } else { + dependencies = []string{} + schedulerID = "-" + } + + payloadBytes, err := json.MarshalIndent(payload, "", " ") + log.Printf("Payload:\n\n%v\n\n", string(payloadBytes)) + if err != nil { + return nil, fmt.Errorf("Cannot convert payload %#v to JSON: %s", payload, err) + } + + created := time.Now() + deadline := created.AddDate(0, 0, 1) + expires := deadline.AddDate(1, 0, 0) + return &tcqueue.TaskDefinitionRequest{ + Created: tcclient.Time(created), + Deadline: tcclient.Time(deadline), + Dependencies: dependencies, + Expires: tcclient.Time(expires), + Metadata: tcqueue.TaskMetadata{ + Description: description, + Name: name, + Owner: "taskcluster-internal@mozilla.com", + Source: "https://github.com/taskcluster/generic-worker", + }, + Payload: json.RawMessage(payloadBytes), + ProvisionerID: provisionerID, + SchedulerID: schedulerID, + Scopes: scopes, + TaskGroupID: dt.TaskGroupID, + WorkerType: workerType, + }, nil +} + +func (tg *TaskGroup) Submit() error { + queue := tcqueue.NewFromEnv() + for taskID, tdr := range tg.taskDefs { + resp, err := queue.CreateTask(taskID, tdr) + if err != nil { + return fmt.Errorf("Error submitting task:\n%#v\n\n%s", *tdr, err) + } + fmt.Printf("Task %v %v...\n", taskID, resp.Status.State) + } + return nil +} diff --git a/gw-decision-task/tasks.yml b/gw-decision-task/tasks.yml new file mode 100644 index 00000000..cf63fdd2 --- /dev/null +++ b/gw-decision-task/tasks.yml @@ -0,0 +1,297 @@ +Types: + + BuildAndTest: + Name: 'Build/test generic-worker (${ENGINE} engine) on ${PLATFORM}' + Description: 'This builds and tests the ${ARCH} version of generic-worker (${ENGINE} engine) on ${PLATFORM}' + Mounts: + - 'go1.10.8' + - 'git2.24.0.2' + - 'jq1.6' + - 'ci-creds' + Command: BuildAndTest + Features: + taskclusterProxy: true + Scopes: + - 'generic-worker:cache:generic-worker-checkout' + - 'secrets:get:project/taskcluster/testing/generic-worker/ci-creds' + Artifacts: + - Name: 'public/build/generic-worker-${OS}-${ARCH}${EXTENSION}' + Path: 'gopath1.10.8/bin/generic-worker${EXTENSION}' + Type: 'file' + MaxRunTime: 3600 + + CheckDeps: + Name: 'Run `dep check` to ensure golang vendored dependencies are up-to-date' + Description: | + Run `dep check` to ensure golang vendored dependencies are up-to-date. + See [go dep docs](https://golang.github.io/dep/docs/daily-dep.html#key-takeaways) + for more information. + Mounts: + - 'go1.10.8' + Command: CheckDeps + MaxRunTime: 3600 + +Tasks: + BuildAndTest: + - WorkerPool: 'proj-taskcluster/gw-ci-macos' + Env: + ENGINE: 'multiuser' + - WorkerPool: 'proj-taskcluster/gw-ci-macos' + Env: + ENGINE: 'simple' + - WorkerPool: 'proj-taskcluster/gw-ci-raspbian-stretch' + Env: + ENGINE: 'simple' + - WorkerPool: 'proj-taskcluster/gw-ci-ubuntu-18-04' + Env: + ENGINE: 'multiuser' + - WorkerPool: 'proj-taskcluster/gw-ci-ubuntu-18-04' + Env: + ENGINE: 'simple' + - WorkerPool: 'proj-taskcluster/gw-ci-ubuntu-18-04' + Env: + ENGINE: 'docker' + - WorkerPool: 'proj-taskcluster/gw-ci-windows10-amd64' + Env: + ENGINE: 'multiuser' + # We must set here since this worker pool does not have a Z: drive + GW_SKIP_Z_DRIVE_TESTS: 'true' + # This worker pool has no mozilla-build installation + GW_SKIP_MOZILLA_BUILD_TESTS: 'true' + # This worker pool has no python installation + GW_SKIP_PYTHON_TESTS: 'true' + - WorkerPool: 'proj-taskcluster/gw-ci-windows10-arm' + Env: + ENGINE: 'multiuser' + # We must set here since this worker pool does not have a Z: drive + GW_SKIP_Z_DRIVE_TESTS: 'true' + # This worker pool has no mozilla-build installation + GW_SKIP_MOZILLA_BUILD_TESTS: 'true' + # This worker pool has no python installation + GW_SKIP_PYTHON_TESTS: 'true' + - WorkerPool: 'proj-taskcluster/gw-ci-windows2012r2-amd64' + Env: + ENGINE: 'multiuser' + # We must set here since this worker type does not have a Z: drive + GW_SKIP_Z_DRIVE_TESTS: 'true' +##################################################################### +# ############################################################# # +# # DISABLING WINDOWS 7 UNTIL WE HAVE WINDOWS 7 WORKERS AGAIN # # +# ############################################################# # +# - WorkerPool: 'proj-taskcluster/gw-ci-windows7-386' # +# Env: # +# ENGINE: 'multiuser' # +##################################################################### + CheckDeps: + - WorkerPool: 'proj-taskcluster/gw-ci-ubuntu-18-04' + +WorkerPools: + proj-taskcluster/gw-ci-macos: + Platform: 'macOS Mojave 10.14' + OS: 'darwin' + Arch: 'amd64' + proj-taskcluster/gw-ci-raspbian-stretch: + Platform: 'Raspbian GNU/Linux 9 (stretch)' + OS: 'linux' + Arch: 'armv6l' + proj-taskcluster/gw-ci-ubuntu-18-04: + Platform: 'Ubuntu 18.04 (amd64)' + OS: 'linux' + Arch: 'amd64' + proj-taskcluster/gw-ci-windows10-amd64: + Platform: 'Windows 10 (amd64)' + OS: 'windows' + Arch: 'amd64' + proj-taskcluster/gw-ci-windows10-arm: + Platform: 'Windows 10 (arm)' + OS: 'windows' + # There is no arm release for go 1.10.8 on windows, but 386 release works + # through emulation provided by the host OS. + Arch: '386' + proj-taskcluster/gw-ci-windows2012r2-amd64: + Platform: 'Windows Server 2012 R2 (amd64)' + OS: 'windows' + Arch: 'amd64' + proj-taskcluster/gw-ci-windows7-386: + Platform: 'Windows 7 (386)' + OS: 'windows' + Arch: '386' + +Commands: + CheckDeps: + Posix: + - - /bin/bash + - -vxec + - | + export CGO_ENABLED=0 + export GOROOT="$(pwd)/go1.10.8/go" + export GOPATH="$(pwd)/gopath1.10.8" + export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" + go version + go env + mkdir -p "${GOPATH}/src/github.com/taskcluster" + cd "${GOPATH}/src/github.com/taskcluster" + if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone "${GITHUB_CLONE_URL}" 'generic-worker'; fi + cd 'generic-worker' + git fetch "${GITHUB_CLONE_URL}" "+${GITHUB_SHA}:refs/heads/X${TASK_ID}" + git checkout -f "X${TASK_ID}" + git reset --hard "${GITHUB_SHA}" + git clean -fdx + git checkout -B tmp -t "X${TASK_ID}" + mkdir -p "${GOPATH}/bin" + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + dep check + + BuildAndTest: + Posix: + - - /bin/bash + - -vxec + - | + function b64 { + [ "$(uname -s)" != "Darwin" ] || base64 -D + [ "$(uname -s)" != "Linux" ] || base64 -d + } + # go test: -race and -msan are only supported on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64 + test $(uname -m) == "x86_64" && RACE=-race || RACE= + export CGO_ENABLED=0 + export GOROOT="$(pwd)/go1.10.8/go" + export GOPATH="$(pwd)/gopath1.10.8" + export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" + git --version + go version + go env + curl -s "${TASKCLUSTER_PROXY_URL}/secrets/v1/secret/project/taskcluster/testing/generic-worker/ci-creds" | sed -n 's/.*"b64_encoded_credentials_script": "\(.*\)".*/\1/p' | b64 > tc-creds.sh + source tc-creds.sh + mkdir -p "${GOPATH}/src/github.com/taskcluster" + cd "${GOPATH}/src/github.com/taskcluster" + if [ ! -d generic-worker/.git ]; then rm -rf generic-worker; git clone "${GITHUB_CLONE_URL}" 'generic-worker'; fi + cd 'generic-worker' + git fetch "${GITHUB_CLONE_URL}" "+${GITHUB_SHA}:refs/heads/X${TASK_ID}" + git checkout -f "X${TASK_ID}" + git reset --hard "${GITHUB_SHA}" + git clean -fdx + git checkout -B tmp -t "X${TASK_ID}" + go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign + cd gw-codegen + go get -v + cd .. + go generate + go install -tags "${ENGINE}" -v -ldflags "-X main.revision=${GITHUB_SHA}" ./... + # output of wc command can contain spaces on darwin, so no quotes around expression + test $(git status --porcelain | wc -l) == 0 + if [ "${ENGINE}" == "multiuser" ]; then + cp "${TASK_USER_CREDENTIALS}" next-task-user.json + # IMPORTANT - run go test with GW_TESTS_RUN_AS_CURRENT_USER=true *before* running it without + # otherwise tests that call `go run ....` will write go object files to .cache as root + GW_TESTS_RUN_AS_CURRENT_USER=true GORACE=history_size=7 CGO_ENABLED=1 go test -tags "${ENGINE}" -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=${GITHUB_SHA}" -v ${RACE} + fi + if [ "$(go env GOARCH)" != 'armv6l' ]; then + export CGO_ENABLED=1 + fi + GORACE=history_size=7 go test -tags "${ENGINE}" -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=${GITHUB_SHA}" -v ${RACE} ./... + "${GOPATH}/bin/ineffassign" . + Windows: + - | + :: go test: -race and -msan are only supported on linux/amd64, freebsd/amd64, darwin/amd64 and windows/amd64 + reg Query "HKLM\Hardware\Description\System\CentralProcessor\0" | find /i "Intel64" > NUL && set RACE=-race || set "RACE= " + :: find.exe may have exited with exit code 1, so need to explicitly exit with 0 + exit /b 0 + - jq -r .secret.b64_encoded_credentials_batch_script ci-creds.json > tc-creds.bat.b64 + - certutil -decode tc-creds.bat.b64 tc-creds.bat + - call tc-creds.bat 2>&1 + - set CGO_ENABLED=0 + - set GOPATH=%CD%\gopath1.10.8 + - set GOROOT=%CD%\go1.10.8\go + - set PATH=%CD%\git\cmd;%GOPATH%\bin;%GOROOT%\bin;%PATH% + - git version + - go version + - go env + - git config --global core.autocrlf false + - 'if not exist "%GOPATH%\src\github.com\taskcluster" mkdir "%GOPATH%\src\github.com\taskcluster"' + - 'cd "%GOPATH%\src\github.com\taskcluster"' + - 'if not exist generic-worker git clone %GITHUB_CLONE_URL% generic-worker' + - 'cd generic-worker' + - 'git fetch %GITHUB_CLONE_URL% +%GITHUB_SHA%:refs/heads/X%TASK_ID%' + - 'git checkout -f "X%TASK_ID%"' + - 'git reset --hard %GITHUB_SHA%' + - 'git clean -fdx' + - 'git checkout -B tmp -t "X%TASK_ID%"' + - go get -v -u github.com/taskcluster/livelog github.com/taskcluster/taskcluster-proxy github.com/gordonklaus/ineffassign + - cd gw-codegen + - go get -v + - cd .. + - go generate + - | + :: this counts the number of lines returned by git status + :: dump temp file a directory higher, otherwise git status reports the tmp1.txt file! + git status --porcelain | C:\Windows\System32\find.exe /v /c "" > ..\tmp1.txt + set /P lines=<..\tmp1.txt + :: this checks that if more than 0 lines are returned, we fail + if %lines% gtr 0 exit /b 64 + :: find.exe may have exited with exit code 1, so need to explicitly exit with 0 + exit /b 0 + - go install -tags "%ENGINE%" -v -ldflags "-X main.revision=%GITHUB_SHA%" ./... + - set CGO_ENABLED=%CGO_ENABLED_TESTS% + - set GORACE=history_size=7 + - copy "%TASK_USER_CREDENTIALS%" "%CD%\next-task-user.json" + - 'go test -tags "%ENGINE%" -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=%GITHUB_SHA%" -v %RACE% ./...' + - set GW_TESTS_RUN_AS_CURRENT_USER=true + - 'go test -tags "%ENGINE%" -timeout 45m -ldflags "-X github.com/taskcluster/generic-worker.revision=%GITHUB_SHA%" -v %RACE%' + - ineffassign . + +Mounts: + go1.10.8: + directory: go1.10.8 + content: + darwin: + amd64: + url: 'https://storage.googleapis.com/golang/go1.10.8.darwin-amd64.tar.gz' + sha256: 'f41bc914a721ac98a187df824b3b40f0a7f35bfb3c6d31221bdd940d537d3c28' + format: tar.gz + linux: + armv6l: + url: 'https://storage.googleapis.com/golang/go1.10.8.linux-armv6l.tar.gz' + sha256: '6fdbc67524fc4c15fc87014869dddce9ecda7958b78f3cb1bbc5b0a9b61bfb95' + format: tar.gz + amd64: + url: 'https://storage.googleapis.com/golang/go1.10.8.linux-amd64.tar.gz' + sha256: 'd8626fb6f9a3ab397d88c483b576be41fa81eefcec2fd18562c87626dbb3c39e' + format: tar.gz + windows: + 386: + url: 'https://storage.googleapis.com/golang/go1.10.8.windows-386.zip' + sha256: '9ded97d830bef3734ea6de70df0159656d6a63e01484175b34d72b8db326bda0' + format: zip + amd64: + url: 'https://storage.googleapis.com/golang/go1.10.8.windows-amd64.zip' + sha256: 'ab63b55c349f75cce4b93aefa9b52828f50ebafb302da5057db0e686d7873d7a' + format: zip + git2.24.0.2: + directory: git + content: + windows: + 386: + url: 'https://github.com/git-for-windows/git/releases/download/v2.24.0.windows.2/MinGit-2.24.0.2-32-bit.zip' + sha256: 'b7b26e87d3df9b44fee0606a9df3ca839bc444faf76c4c27bc0824a6c10e7831' + format: zip + amd64: + url: 'https://github.com/git-for-windows/git/releases/download/v2.24.0.windows.2/MinGit-2.24.0.2-64-bit.zip' + sha256: 'c33aec6ae68989103653ca9fb64f12cabccf6c61d0dde30c50da47fc15cf66e2' + format: zip + jq1.6: + file: jq.exe + content: + windows: + 386: + url: 'https://github.com/stedolan/jq/releases/download/jq-1.6/jq-win32.exe' + sha256: '0012cb4c0eb6eaf97b842e676e423a69a8fea95055d93830551b4a5a54494bd8' + amd64: + url: 'https://github.com/stedolan/jq/releases/download/jq-1.6/jq-win64.exe' + sha256: 'a51d36968dcbdeabb3142c6f5cf9b401a65dc3a095f3144bd0c118d5bb192753' + ci-creds: + file: ci-creds.json + content: + windows: + all: + url: 'http://localhost/secrets/v1/secret/project/taskcluster/testing/generic-worker/ci-creds' + sha256: 'ef1d954dbae01a0810fcdb56f56761dbbb26692b0dc8cbb2994101512fc26992' diff --git a/helper_test.go b/helper_test.go index c39b18a3..e838013c 100644 --- a/helper_test.go +++ b/helper_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/pborman/uuid" "github.com/taskcluster/generic-worker/gwconfig" "github.com/taskcluster/generic-worker/testutil" "github.com/taskcluster/httpbackoff" @@ -202,6 +203,11 @@ func NewQueue(t *testing.T) *tcqueue.Queue { func scheduleTask(t *testing.T, td *tcqueue.TaskDefinitionRequest, payload GenericWorkerPayload) (taskID string) { taskID = slugid.Nice() + scheduleNamedTask(t, td, payload, taskID) + return +} + +func scheduleNamedTask(t *testing.T, td *tcqueue.TaskDefinitionRequest, payload GenericWorkerPayload, taskID string) { if td.Payload == nil { b, err := json.Marshal(&payload) @@ -231,8 +237,6 @@ func scheduleTask(t *testing.T, td *tcqueue.TaskDefinitionRequest, payload Gener t.Fatalf("Could not submit task: %v", err) } t.Logf("Scheduled task %v", taskID) - - return } func execute(t *testing.T, expectedExitCode ExitCode) { @@ -483,3 +487,95 @@ func cancelTask(t *testing.T) (td *tcqueue.TaskDefinitionRequest, payload Generi } return } + +// CreateArtifactFromFile returns a taskID for a task with an artifact with the +// given name whose content matches the content of the local file (relative to +// the testdata folder) with the given path. It does this by creating a hash of +// the file content together with the name of the file, and then converts the +// hash into a "nice" slug. It then checks if the task already exists. If it +// does exist, it simply returns the taskID. If it doesn't, it creates the task +// and returns. +func CreateArtifactFromFile(t *testing.T, path string, name string) (taskID string) { + + // Calculate hash of file content + rawContent, err := os.Open(filepath.Join(testdataDir, path)) + if err != nil { + t.Fatal(err) + } + defer rawContent.Close() + hasher := sha256.New() + _, err = io.Copy(hasher, rawContent) + if err != nil { + t.Fatal(err) + } + + // Append a 0 byte and the artifact name to the hash source, to ensure that + // if the artifact name changes, we get a different taskID. Since file + // names can't include a 0 byte, adding the zero byte as a separator + // between the two parts ensures that a one-to-one mapping exists between + // {file content, artifact name} and hash. + _, err = hasher.Write(append([]byte{0}, []byte(name)...)) + if err != nil { + t.Fatal(err) + } + + // Use first 128 bits of 256 bit hash for UUID + sha256 := hasher.Sum(nil) + v4uuid := sha256[:16] + + // Comply to uuid v4 rules (mask six bits) + v4uuid[6] = (v4uuid[6] & 0x0f) | 0x40 // Version 4 + v4uuid[8] = (v4uuid[8] & 0x3f) | 0x80 // Variant is 10 + + // Make slugid a "nice" one (mask one further bit => 121 bits entropy) + v4uuid[0] &= 0x7f + + // Convert to a string taskID + taskID = slugid.Encode(uuid.UUID(v4uuid)) + + // See if task already exists + tdr, err := testQueue.Task(taskID) + if err != nil { + switch e := err.(type) { + case *tcclient.APICallException: + switch r := e.RootCause.(type) { + case httpbackoff.BadHttpResponseCode: + if r.HttpResponseCode == 404 { + t.Logf("Creating task %q for artifact %v under path %v...", taskID, name, path) + payload := GenericWorkerPayload{ + Command: copyTestdataFile(path), + MaxRunTime: 30, + Artifacts: []Artifact{ + { + Path: path, + Name: name, + Type: "file", + }, + }, + } + td := testTask(t) + // Set 6 month expiry + td.Expires = tcclient.Time(time.Now().AddDate(0, 6, 0)) + td.Metadata.Name = "Task dependency for generic-worker integration tests" + td.Metadata.Description = fmt.Sprintf("Single artifact %v from path %v with hash %v", name, path, hex.EncodeToString(sha256)) + scheduleNamedTask(t, td, payload, taskID) + ensureResolution(t, taskID, "completed", "completed") + return + } + } + } + t.Fatalf("%#v", err) + } + + // If task expires in the next two minutes, just fail intentionally. It + // isn't worth trying to handle this situation, since the task only expires + // after 6 months, so the chance of hitting the two minute period before it + // expires is extremely small, and the error will explicitly report it + // anyway. + remainingTime := time.Time(tdr.Expires).Sub(time.Now()) + if remainingTime.Seconds() < 120 { + t.Fatalf("You've been extremely unlucky. This test depends on task %q that was created six months ago but is due to expire in less than two minutes (%v). Wait a few minutes and try again!", taskID, remainingTime) + } + t.Logf("Depend on task %q which expires in %v.", taskID, remainingTime) + return +} diff --git a/main.go b/main.go index e6cd4804..1d556009 100644 --- a/main.go +++ b/main.go @@ -950,7 +950,7 @@ func (task *TaskRun) Run() (err *ExecutionErrors) { if err.Occurred() { return } - log.Printf("Running task https://tools.taskcluster.net/task-inspector/#%v/%v", task.TaskID, task.RunID) + log.Printf("Running task %v/tasks/%v/runs/%v", config.RootURL, task.TaskID, task.RunID) task.Commands = make([]*process.Command, len(task.Payload.Command)) // generate commands, in case features want to modify them diff --git a/main_broken_on_docker_test.go b/main_broken_on_docker_test.go index d1eb5070..270182c7 100644 --- a/main_broken_on_docker_test.go +++ b/main_broken_on_docker_test.go @@ -16,9 +16,8 @@ import ( func TestAbortAfterMaxRunTime(t *testing.T) { defer setup(t)() - // include a writable directory cache where our process writes to, to make - // sure we are still able unmount cache when we abort process prematurely - // that is writing to the cache + // Include a writable directory cache, to test that caches can be unmounted + // when a task aborts prematurely. mounts := []MountEntry{ // requires scope "generic-worker:cache:banana-cache" &WritableDirectoryCache{ diff --git a/mounts_broken_on_docker_test.go b/mounts_broken_on_docker_test.go index dbaf92c5..cb70a1f5 100644 --- a/mounts_broken_on_docker_test.go +++ b/mounts_broken_on_docker_test.go @@ -15,14 +15,17 @@ func TestMounts(t *testing.T) { defer setup(t)() + taskID1 := CreateArtifactFromFile(t, "SampleArtifacts/_/X.txt", "SampleArtifacts/_/X.txt") + taskID2 := CreateArtifactFromFile(t, "mozharness.zip", "public/build/mozharness.zip") + taskID3 := CreateArtifactFromFile(t, "unknown_issuer_app_1.zip", "public/build/unknown_issuer_app_1.zip") + mounts := []MountEntry{ // file mount from artifact &FileMount{ File: filepath.Join("preloaded", "Mr X.txt"), - // Note: the task definition for taskId KTBKfEgxR5GdfIIREQIvFQ can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "KTBKfEgxR5GdfIIREQIvFQ", + "taskId": "` + taskID1 + `", "artifact": "SampleArtifacts/_/X.txt" }`), }, @@ -73,9 +76,8 @@ func TestMounts(t *testing.T) { &WritableDirectoryCache{ CacheName: "unknown-issuer-app-cache", Directory: filepath.Join("my-task-caches", "unknown_issuer_app_1"), - // Note: the task definition for taskId LK1Rz2UtT16d-HBSqyCtuA can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "LK1Rz2UtT16d-HBSqyCtuA", + "taskId": "` + taskID3 + `", "artifact": "public/build/unknown_issuer_app_1.zip" }`), Format: "zip", @@ -94,9 +96,8 @@ func TestMounts(t *testing.T) { // read only directory from artifact &ReadOnlyDirectory{ Directory: filepath.Join("my-task-caches", "mozharness"), - // Note: the task definition for taskId VESwp9JaRo-XkFN_bemBhw can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "VESwp9JaRo-XkFN_bemBhw", + "taskId": "` + taskID2 + `", "artifact": "public/build/mozharness.zip" }`), Format: "zip", @@ -126,9 +127,9 @@ func TestMounts(t *testing.T) { td := testTask(t) td.Dependencies = []string{ - "KTBKfEgxR5GdfIIREQIvFQ", - "LK1Rz2UtT16d-HBSqyCtuA", - "VESwp9JaRo-XkFN_bemBhw", + taskID1, + taskID2, + taskID3, } td.Scopes = []string{ "queue:get-artifact:SampleArtifacts/_/X.txt", @@ -143,8 +144,7 @@ func TestMounts(t *testing.T) { checkSHA256( t, "625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e", - // Note: the task definition for taskId LK1Rz2UtT16d-HBSqyCtuA can be seen in the testdata/tasks directory - fileCaches["artifact:LK1Rz2UtT16d-HBSqyCtuA:public/build/unknown_issuer_app_1.zip"].Location, + fileCaches["artifact:"+taskID3+":public/build/unknown_issuer_app_1.zip"].Location, ) checkSHA256( t, @@ -154,8 +154,7 @@ func TestMounts(t *testing.T) { checkSHA256( t, "8308d593eb56527137532595a60255a3fcfbe4b6b068e29b22d99742bad80f6f", - // Note: the task definition for taskId KTBKfEgxR5GdfIIREQIvFQ can be seen in the testdata/tasks directory - fileCaches["artifact:KTBKfEgxR5GdfIIREQIvFQ:SampleArtifacts/_/X.txt"].Location, + fileCaches["artifact:"+taskID1+":SampleArtifacts/_/X.txt"].Location, ) checkSHA256( t, @@ -165,8 +164,7 @@ func TestMounts(t *testing.T) { checkSHA256( t, "613193e90dcba442ffa01622834387bb5f175fdc67c46f564284261076994a75", - // Note: the task definition for taskId VESwp9JaRo-XkFN_bemBhw can be seen in the testdata/tasks directory - fileCaches["artifact:VESwp9JaRo-XkFN_bemBhw:public/build/mozharness.zip"].Location, + fileCaches["artifact:"+taskID2+":public/build/mozharness.zip"].Location, ) checkSHA256( t, @@ -237,6 +235,8 @@ func TestCachesCanBeModified(t *testing.T) { // TestCacheMoved tests that if a test mounts a cache, and then moves it to a // different location, that the test fails, and the worker doesn't crash. func TestCacheMoved(t *testing.T) { + defer setup(t)() + taskID := CreateArtifactFromFile(t, "unknown_issuer_app_1.zip", "public/build/unknown_issuer_app_1.zip") // whether permission is granted to task user depends if running under windows or not // and is independent of whether running as current user or not @@ -245,9 +245,9 @@ func TestCacheMoved(t *testing.T) { // No cache on first pass pass1 := append([]string{ `No existing writable directory cache 'banana-cache' - creating .*`, - `Downloading task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Content from task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip \(.*\) matches required SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`, + `Downloading task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Content from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip \(.*\) matches required SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`, `Creating directory .*` + t.Name() + ` with permissions 0700`, `Extracting zip file .* to '.*` + t.Name() + `'`, }, @@ -258,13 +258,13 @@ func TestCacheMoved(t *testing.T) { `Preserving cache: Moving ".*`+t.Name()+`" to ".*"`, `Removing cache banana-cache from cache table`, `Deleting cache banana-cache file\(s\) at .*`, - `Could not unmount task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip due to: 'Could not persist cache "banana-cache" due to .*'`, + `Could not unmount task `+taskID+` artifact public/build/unknown_issuer_app_1.zip due to: 'Could not persist cache "banana-cache" due to .*'`, ) // On second pass, cache already exists pass2 := append([]string{ `No existing writable directory cache 'banana-cache' - creating .*`, - `Found existing download for artifact:LK1Rz2UtT16d-HBSqyCtuA:public/build/unknown_issuer_app_1.zip \(.*\) with correct SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`, + `Found existing download for artifact:` + taskID + `:public/build/unknown_issuer_app_1.zip \(.*\) with correct SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`, `Creating directory .*` + t.Name() + ` with permissions 0700`, `Extracting zip file .* to '.*` + t.Name() + `'`, }, @@ -275,7 +275,7 @@ func TestCacheMoved(t *testing.T) { `Preserving cache: Moving ".*`+t.Name()+`" to ".*"`, `Removing cache banana-cache from cache table`, `Deleting cache banana-cache file\(s\) at .*`, - `Could not unmount task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip due to: 'Could not persist cache "banana-cache" due to .*'`, + `Could not unmount task `+taskID+` artifact public/build/unknown_issuer_app_1.zip due to: 'Could not persist cache "banana-cache" due to .*'`, ) LogTest( @@ -285,9 +285,8 @@ func TestCacheMoved(t *testing.T) { &WritableDirectoryCache{ CacheName: "banana-cache", Directory: t.Name(), - // Note: the task definition for taskId LK1Rz2UtT16d-HBSqyCtuA can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "LK1Rz2UtT16d-HBSqyCtuA", + "taskId": "` + taskID + `", "artifact": "public/build/unknown_issuer_app_1.zip", "sha256": "625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e" }`), @@ -295,7 +294,7 @@ func TestCacheMoved(t *testing.T) { }, }, Dependencies: []string{ - "LK1Rz2UtT16d-HBSqyCtuA", + taskID, }, Scopes: []string{"generic-worker:cache:banana-cache"}, Payload: &GenericWorkerPayload{ @@ -317,15 +316,18 @@ func TestCacheMoved(t *testing.T) { func TestMountFileAndDirSameLocation(t *testing.T) { + defer setup(t)() + taskID := CreateArtifactFromFile(t, "unknown_issuer_app_1.zip", "public/build/unknown_issuer_app_1.zip") + // whether permission is granted to task user depends if running under windows or not // and is independent of whether running as current user or not granting, _ := grantingDenying(t, "file", "file-located-here") // No cache on first pass pass1 := append([]string{ - `Downloading task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Download .* of task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task payload does not declare a required value, so content authenticity cannot be verified`, + `Downloading task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Download .* of task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task payload does not declare a required value, so content authenticity cannot be verified`, `Creating directory .* with permissions 0700`, `Copying .* to .*file-located-here`, }, @@ -333,7 +335,7 @@ func TestMountFileAndDirSameLocation(t *testing.T) { ) pass1 = append(pass1, - `Found existing download for artifact:LK1Rz2UtT16d-HBSqyCtuA:public/build/unknown_issuer_app_1.zip \(.*\) with correct SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`, + `Found existing download for artifact:`+taskID+`:public/build/unknown_issuer_app_1.zip \(.*\) with correct SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`, `Creating directory .*file-located-here with permissions 0700`, // error is platform specific `(mkdir .*file-located-here: not a directory|mkdir .*file-located-here: The system cannot find the path specified.|Cannot create directory .*file-located-here)`, @@ -341,7 +343,7 @@ func TestMountFileAndDirSameLocation(t *testing.T) { // On second pass, cache already exists pass2 := append([]string{ - `No SHA256 specified in task mounts for artifact:LK1Rz2UtT16d-HBSqyCtuA:public/build/unknown_issuer_app_1.zip - SHA256 from downloaded file .* is 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e.`, + `No SHA256 specified in task mounts for artifact:` + taskID + `:public/build/unknown_issuer_app_1.zip - SHA256 from downloaded file .* is 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e.`, `Creating directory .* with permissions 0700`, `Copying .* to .*file-located-here`, }, @@ -349,7 +351,7 @@ func TestMountFileAndDirSameLocation(t *testing.T) { ) pass2 = append(pass2, - `Found existing download for artifact:LK1Rz2UtT16d-HBSqyCtuA:public/build/unknown_issuer_app_1.zip \(.*\) with correct SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`, + `Found existing download for artifact:`+taskID+`:public/build/unknown_issuer_app_1.zip \(.*\) with correct SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`, `Creating directory .*file-located-here with permissions 0700`, // error is platform specific `(mkdir .*file-located-here: not a directory|mkdir .*file-located-here: The system cannot find the path specified.|Cannot create directory .*file-located-here)`, @@ -361,17 +363,15 @@ func TestMountFileAndDirSameLocation(t *testing.T) { Mounts: []MountEntry{ &FileMount{ File: "file-located-here", - // Note: the task definition for taskId LK1Rz2UtT16d-HBSqyCtuA can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "LK1Rz2UtT16d-HBSqyCtuA", + "taskId": "` + taskID + `", "artifact": "public/build/unknown_issuer_app_1.zip" }`), }, &ReadOnlyDirectory{ Directory: "file-located-here", - // Note: the task definition for taskId LK1Rz2UtT16d-HBSqyCtuA can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "LK1Rz2UtT16d-HBSqyCtuA", + "taskId": "` + taskID + `", "artifact": "public/build/unknown_issuer_app_1.zip", "sha256": "625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e" }`), @@ -379,7 +379,7 @@ func TestMountFileAndDirSameLocation(t *testing.T) { }, }, Dependencies: []string{ - "LK1Rz2UtT16d-HBSqyCtuA", + taskID, }, TaskRunResolutionState: "failed", TaskRunReasonResolved: "failed", @@ -396,6 +396,7 @@ func TestMountFileAndDirSameLocation(t *testing.T) { func TestInvalidSHADoesNotPreventMountedMountsFromBeingUnmounted(t *testing.T) { defer setup(t)() + taskID := CreateArtifactFromFile(t, "unknown_issuer_app_1.zip", "public/build/unknown_issuer_app_1.zip") mounts := []MountEntry{ &WritableDirectoryCache{ @@ -404,10 +405,9 @@ func TestInvalidSHADoesNotPreventMountedMountsFromBeingUnmounted(t *testing.T) { }, &ReadOnlyDirectory{ Directory: filepath.Join(t.Name(), "2"), - // Note: the task definition for taskId LK1Rz2UtT16d-HBSqyCtuA can be seen in the testdata/tasks directory // SHA256 is intentionally incorrect to make sure that above cache is still persisted Content: json.RawMessage(`{ - "taskId": "LK1Rz2UtT16d-HBSqyCtuA", + "taskId": "` + taskID + `", "artifact": "public/build/unknown_issuer_app_1.zip", "sha256": "7777777777777777777777777777777777777777777777777777777777777777" }`), @@ -423,7 +423,7 @@ func TestInvalidSHADoesNotPreventMountedMountsFromBeingUnmounted(t *testing.T) { td := testTask(t) td.Dependencies = []string{ - "LK1Rz2UtT16d-HBSqyCtuA", + taskID, } td.Scopes = []string{ "generic-worker:cache:unknown-issuer-app-cache", diff --git a/mounts_test.go b/mounts_test.go index 216de306..042db1c1 100644 --- a/mounts_test.go +++ b/mounts_test.go @@ -10,17 +10,21 @@ import ( "testing" "github.com/taskcluster/generic-worker/gwconfig" + "github.com/taskcluster/slugid-go/slugid" ) func TestMissingScopes(t *testing.T) { defer setup(t)() + + taskID := CreateArtifactFromFile(t, "SampleArtifacts/_/X.txt", "SampleArtifacts/_/X.txt") + + // Create a new task to mount the artifact without the scope to do so mounts := []MountEntry{ // requires scope "queue:get-artifact:SampleArtifacts/_/X.txt" &FileMount{ File: filepath.Join("preloaded", "Mr X.txt"), - // Note: the task definition for taskId KTBKfEgxR5GdfIIREQIvFQ can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "KTBKfEgxR5GdfIIREQIvFQ", + "taskId": "` + taskID + `", "artifact": "SampleArtifacts/_/X.txt" }`), }, @@ -39,7 +43,7 @@ func TestMissingScopes(t *testing.T) { td := testTask(t) td.Dependencies = []string{ - "KTBKfEgxR5GdfIIREQIvFQ", + taskID, } // don't set any scopes @@ -59,13 +63,14 @@ func TestMissingScopes(t *testing.T) { // TestMissingDependency tests that if artifact content is mounted, it must be included as a task dependency func TestMissingMountsDependency(t *testing.T) { defer setup(t)() + pretendTaskID := slugid.Nice() mounts := []MountEntry{ // requires scope "queue:get-artifact:SampleArtifacts/_/X.txt" &FileMount{ File: filepath.Join("preloaded", "Mr X.txt"), - // Note: the task definition for taskId KTBKfEgxR5GdfIIREQIvFQ can be seen in the testdata/tasks directory + // Pretend task Content: json.RawMessage(`{ - "taskId": "KTBKfEgxR5GdfIIREQIvFQ", + "taskId": "` + pretendTaskID + `", "artifact": "SampleArtifacts/_/X.txt" }`), }, @@ -96,7 +101,7 @@ func TestMissingMountsDependency(t *testing.T) { t.Fatalf("Error when trying to read log file: %v", err) } logtext := string(bytes) - if !strings.Contains(logtext, "[mounts] task.dependencies needs to include KTBKfEgxR5GdfIIREQIvFQ since one or more of its artifacts are mounted") { + if !strings.Contains(logtext, "[mounts] task.dependencies needs to include "+pretendTaskID+" since one or more of its artifacts are mounted") { t.Fatalf("Was expecting log file to explain that task dependency was missing, but it doesn't: \n%v", logtext) } } @@ -114,13 +119,15 @@ func Test32BitOverflow(t *testing.T) { func TestCorruptZipDoesntCrashWorker(t *testing.T) { defer setup(t)() + + taskID := CreateArtifactFromFile(t, "SampleArtifacts/_/X.txt", "SampleArtifacts/_/X.txt") + mounts := []MountEntry{ // requires scope "queue:get-artifact:SampleArtifacts/_/X.txt" &ReadOnlyDirectory{ Directory: ".", - // Note: the task definition for taskId KTBKfEgxR5GdfIIREQIvFQ can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "KTBKfEgxR5GdfIIREQIvFQ", + "taskId": "` + taskID + `", "artifact": "SampleArtifacts/_/X.txt" }`), Format: "zip", @@ -135,7 +142,7 @@ func TestCorruptZipDoesntCrashWorker(t *testing.T) { td := testTask(t) td.Dependencies = []string{ - "KTBKfEgxR5GdfIIREQIvFQ", + taskID, } td.Scopes = []string{"queue:get-artifact:SampleArtifacts/_/X.txt"} @@ -164,13 +171,15 @@ func TestCorruptZipDoesntCrashWorker(t *testing.T) { // task that *does* exist. func TestNonExistentArtifact(t *testing.T) { defer setup(t)() + + taskID := CreateArtifactFromFile(t, "SampleArtifacts/_/X.txt", "SampleArtifacts/_/X.txt") + mounts := []MountEntry{ // requires scope "queue:get-artifact:SampleArtifacts/_/X.txt" &ReadOnlyDirectory{ Directory: ".", - // Note: the task definition for taskId KTBKfEgxR5GdfIIREQIvFQ can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "KTBKfEgxR5GdfIIREQIvFQ", + "taskId": "` + taskID + `", "artifact": "SampleArtifacts/_/non-existent-artifact.txt" }`), Format: "zip", @@ -185,7 +194,7 @@ func TestNonExistentArtifact(t *testing.T) { td := testTask(t) td.Dependencies = []string{ - "KTBKfEgxR5GdfIIREQIvFQ", + taskID, } td.Scopes = []string{"queue:get-artifact:SampleArtifacts/_/non-existent-artifact.txt"} @@ -197,7 +206,7 @@ func TestNonExistentArtifact(t *testing.T) { t.Fatalf("Error when trying to read log file: %v", err) } logtext := string(bytes) - if !strings.Contains(logtext, "[mounts] Could not fetch from task KTBKfEgxR5GdfIIREQIvFQ artifact SampleArtifacts/_/non-existent-artifact.txt into file") { + if !strings.Contains(logtext, "[mounts] Could not fetch from task "+taskID+" artifact SampleArtifacts/_/non-existent-artifact.txt into file") { t.Fatalf("Log did not contain expected text:\n%v", logtext) } } @@ -223,8 +232,6 @@ type MountsLoggingTestCase struct { // This is an extremely strict test helper, that requires you to specify // extracts from every log line that the mounts feature writes to the log func LogTest(m *MountsLoggingTestCase) { - defer setup(m.Test)() - payload := m.Payload if payload == nil { payload = &GenericWorkerPayload{ @@ -279,15 +286,16 @@ func LogTest(m *MountsLoggingTestCase) { } func TestInvalidSHA256(t *testing.T) { + defer setup(t)() + taskID := CreateArtifactFromFile(t, "unknown_issuer_app_1.zip", "public/build/unknown_issuer_app_1.zip") LogTest( &MountsLoggingTestCase{ Test: t, Mounts: []MountEntry{ &ReadOnlyDirectory{ Directory: "unknown_issuer_app_1", - // Note: the task definition for taskId LK1Rz2UtT16d-HBSqyCtuA can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "LK1Rz2UtT16d-HBSqyCtuA", + "taskId": "` + taskID + `", "artifact": "public/build/unknown_issuer_app_1.zip", "sha256": "9263625672993742f0916f7a22b4d9924ed0327f2e02edd18456c0c4e5876850" }`), @@ -295,26 +303,26 @@ func TestInvalidSHA256(t *testing.T) { }, }, Dependencies: []string{ - "LK1Rz2UtT16d-HBSqyCtuA", + taskID, }, TaskRunResolutionState: "failed", TaskRunReasonResolved: "failed", PerTaskRunLogExcerpts: [][]string{ // Required text from first task with no cached value []string{ - `Downloading task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Removing cache artifact:LK1Rz2UtT16d-HBSqyCtuA:public/build/unknown_issuer_app_1.zip from cache table`, - `Deleting cache artifact:LK1Rz2UtT16d-HBSqyCtuA:public/build/unknown_issuer_app_1.zip file\(s\) at .*`, - `Download .* of task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task definition explicitly requires 9263625672993742f0916f7a22b4d9924ed0327f2e02edd18456c0c4e5876850; not retrying download as there were no connection failures and HTTP response status code was 200`, + `Downloading task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Removing cache artifact:` + taskID + `:public/build/unknown_issuer_app_1.zip from cache table`, + `Deleting cache artifact:` + taskID + `:public/build/unknown_issuer_app_1.zip file\(s\) at .*`, + `Download .* of task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task definition explicitly requires 9263625672993742f0916f7a22b4d9924ed0327f2e02edd18456c0c4e5876850; not retrying download as there were no connection failures and HTTP response status code was 200`, }, // Required text from second task when download is already cached []string{ - `Downloading task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Removing cache artifact:LK1Rz2UtT16d-HBSqyCtuA:public/build/unknown_issuer_app_1.zip from cache table`, - `Deleting cache artifact:LK1Rz2UtT16d-HBSqyCtuA:public/build/unknown_issuer_app_1.zip file\(s\) at .*`, - `Download .* of task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task definition explicitly requires 9263625672993742f0916f7a22b4d9924ed0327f2e02edd18456c0c4e5876850; not retrying download as there were no connection failures and HTTP response status code was 200`, + `Downloading task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Removing cache artifact:` + taskID + `:public/build/unknown_issuer_app_1.zip from cache table`, + `Deleting cache artifact:` + taskID + `:public/build/unknown_issuer_app_1.zip file\(s\) at .*`, + `Download .* of task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task definition explicitly requires 9263625672993742f0916f7a22b4d9924ed0327f2e02edd18456c0c4e5876850; not retrying download as there were no connection failures and HTTP response status code was 200`, }, }, }, @@ -322,6 +330,8 @@ func TestInvalidSHA256(t *testing.T) { } func TestValidSHA256(t *testing.T) { + defer setup(t)() + taskID := CreateArtifactFromFile(t, "unknown_issuer_app_1.zip", "public/build/unknown_issuer_app_1.zip") // whether permission is granted to task user depends if running under windows or not // and is independent of whether running as current user or not @@ -329,9 +339,9 @@ func TestValidSHA256(t *testing.T) { // Required text from first task with no cached value pass1 := append([]string{ - `Downloading task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Content from task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip \(.*\) matches required SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`, + `Downloading task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Content from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip \(.*\) matches required SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`, `Creating directory .*unknown_issuer_app_1 with permissions 0700`, `Extracting zip file .* to '.*unknown_issuer_app_1'`, }, @@ -340,7 +350,7 @@ func TestValidSHA256(t *testing.T) { // Required text from second task when download is already cached pass2 := append([]string{ - `Found existing download for artifact:LK1Rz2UtT16d-HBSqyCtuA:public/build/unknown_issuer_app_1.zip \(.*\) with correct SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`, + `Found existing download for artifact:` + taskID + `:public/build/unknown_issuer_app_1.zip \(.*\) with correct SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e`, `Creating directory .*unknown_issuer_app_1 with permissions 0700`, `Extracting zip file .* to '.*unknown_issuer_app_1'`, }, @@ -353,9 +363,8 @@ func TestValidSHA256(t *testing.T) { Mounts: []MountEntry{ &ReadOnlyDirectory{ Directory: "unknown_issuer_app_1", - // Note: the task definition for taskId LK1Rz2UtT16d-HBSqyCtuA can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "LK1Rz2UtT16d-HBSqyCtuA", + "taskId": "` + taskID + `", "artifact": "public/build/unknown_issuer_app_1.zip", "sha256": "625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e" }`), @@ -363,7 +372,7 @@ func TestValidSHA256(t *testing.T) { }, }, Dependencies: []string{ - "LK1Rz2UtT16d-HBSqyCtuA", + taskID, }, TaskRunResolutionState: "completed", TaskRunReasonResolved: "completed", @@ -376,6 +385,8 @@ func TestValidSHA256(t *testing.T) { } func TestFileMountNoSHA256(t *testing.T) { + defer setup(t)() + taskID := CreateArtifactFromFile(t, "unknown_issuer_app_1.zip", "public/build/unknown_issuer_app_1.zip") // whether permission is granted to task user depends if running under windows or not // and is independent of whether running as current user or not @@ -383,9 +394,9 @@ func TestFileMountNoSHA256(t *testing.T) { // No cache on first pass pass1 := append([]string{ - `Downloading task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Download .* of task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task payload does not declare a required value, so content authenticity cannot be verified`, + `Downloading task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Download .* of task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task payload does not declare a required value, so content authenticity cannot be verified`, `Creating directory .* with permissions 0700`, `Copying .* to .*` + t.Name(), }, @@ -394,7 +405,7 @@ func TestFileMountNoSHA256(t *testing.T) { // On second pass, cache already exists pass2 := append([]string{ - `No SHA256 specified in task mounts for artifact:LK1Rz2UtT16d-HBSqyCtuA:public/build/unknown_issuer_app_1.zip - SHA256 from downloaded file .* is 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e.`, + `No SHA256 specified in task mounts for artifact:` + taskID + `:public/build/unknown_issuer_app_1.zip - SHA256 from downloaded file .* is 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e.`, `Creating directory .* with permissions 0700`, `Copying .* to .*` + t.Name(), }, @@ -407,15 +418,14 @@ func TestFileMountNoSHA256(t *testing.T) { Mounts: []MountEntry{ &FileMount{ File: t.Name(), - // Note: the task definition for taskId LK1Rz2UtT16d-HBSqyCtuA can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "LK1Rz2UtT16d-HBSqyCtuA", + "taskId": "` + taskID + `", "artifact": "public/build/unknown_issuer_app_1.zip" }`), }, }, Dependencies: []string{ - "LK1Rz2UtT16d-HBSqyCtuA", + taskID, }, TaskRunResolutionState: "completed", TaskRunReasonResolved: "completed", @@ -430,6 +440,8 @@ func TestFileMountNoSHA256(t *testing.T) { } func TestMountFileAtCWD(t *testing.T) { + defer setup(t)() + taskID := CreateArtifactFromFile(t, "unknown_issuer_app_1.zip", "public/build/unknown_issuer_app_1.zip") LogTest( &MountsLoggingTestCase{ Test: t, @@ -439,35 +451,34 @@ func TestMountFileAtCWD(t *testing.T) { // intentionally setting the path of a directory (current directory) since this should fail test // since a content can't be mounted at the location of an existing directory (content has no explicit filename) File: ".", - // Note: the task definition for taskId LK1Rz2UtT16d-HBSqyCtuA can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "LK1Rz2UtT16d-HBSqyCtuA", + "taskId": "` + taskID + `", "artifact": "public/build/unknown_issuer_app_1.zip" }`), }, }, Dependencies: []string{ - "LK1Rz2UtT16d-HBSqyCtuA", + taskID, }, TaskRunResolutionState: "failed", TaskRunReasonResolved: "failed", PerTaskRunLogExcerpts: [][]string{ // Required text from first task with no cached value []string{ - `Downloading task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Download .* of task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task payload does not declare a required value, so content authenticity cannot be verified`, + `Downloading task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Download .* of task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task payload does not declare a required value, so content authenticity cannot be verified`, `Creating directory .* with permissions 0700`, `Copying .* to .*`, - `Not able to mount content from task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip at path .*`, + `Not able to mount content from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip at path .*`, `open .*: is a directory`, }, // Required text from second task when download is already cached []string{ - `No SHA256 specified in task mounts for artifact:LK1Rz2UtT16d-HBSqyCtuA:public/build/unknown_issuer_app_1.zip - SHA256 from downloaded file .* is 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e.`, + `No SHA256 specified in task mounts for artifact:` + taskID + `:public/build/unknown_issuer_app_1.zip - SHA256 from downloaded file .* is 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e.`, `Creating directory .* with permissions 0700`, `Copying .* to .*`, - `Not able to mount content from task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip at path .*`, + `Not able to mount content from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip at path .*`, `open .*: is a directory`, }, }, @@ -476,6 +487,8 @@ func TestMountFileAtCWD(t *testing.T) { } func TestWritableDirectoryCacheNoSHA256(t *testing.T) { + defer setup(t)() + taskID := CreateArtifactFromFile(t, "unknown_issuer_app_1.zip", "public/build/unknown_issuer_app_1.zip") // whether permission is granted to task user depends if running under windows or not // and is independent of whether running as current user or not @@ -484,9 +497,9 @@ func TestWritableDirectoryCacheNoSHA256(t *testing.T) { // No cache on first pass pass1 := append([]string{ `No existing writable directory cache 'banana-cache' - creating .*`, - `Downloading task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip to .*`, - `Download .* of task LK1Rz2UtT16d-HBSqyCtuA artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task payload does not declare a required value, so content authenticity cannot be verified`, + `Downloading task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Downloaded 4220 bytes with SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e from task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip to .*`, + `Download .* of task ` + taskID + ` artifact public/build/unknown_issuer_app_1.zip has SHA256 625554ec8ce731e486a5fb904f3331d18cf84a944dd9e40c19550686d4e8492e but task payload does not declare a required value, so content authenticity cannot be verified`, `Creating directory .*` + t.Name() + ` with permissions 0700`, `Extracting zip file .* to '.*` + t.Name() + `'`, }, @@ -518,16 +531,15 @@ func TestWritableDirectoryCacheNoSHA256(t *testing.T) { &WritableDirectoryCache{ CacheName: "banana-cache", Directory: t.Name(), - // Note: the task definition for taskId LK1Rz2UtT16d-HBSqyCtuA can be seen in the testdata/tasks directory Content: json.RawMessage(`{ - "taskId": "LK1Rz2UtT16d-HBSqyCtuA", + "taskId": "` + taskID + `", "artifact": "public/build/unknown_issuer_app_1.zip" }`), Format: "zip", }, }, Dependencies: []string{ - "LK1Rz2UtT16d-HBSqyCtuA", + taskID, }, TaskRunResolutionState: "completed", TaskRunReasonResolved: "completed", diff --git a/multiuser_windows_test.go b/multiuser_windows_test.go index 3c89d4f1..95d4f343 100644 --- a/multiuser_windows_test.go +++ b/multiuser_windows_test.go @@ -62,8 +62,10 @@ func TestAppDataNotShared(t *testing.T) { // Test we don't get weird error: // c:\mozilla-build\msys\bin\bash.exe: *** CreateFileMappingA, Win32 error 0. Terminating. func TestNoCreateFileMappingError(t *testing.T) { + if os.Getenv("GW_SKIP_MOZILLA_BUILD_TESTS") != "" { + t.Skip("Skipping since GW_SKIP_MOZILLA_BUILD_TESTS env var is set") + } defer setup(t)() - if config.RunTasksAsCurrentUser { t.Skip("Not running, since we never want to call msys directly from LocalSystem account") } @@ -94,6 +96,9 @@ func TestNoCreateFileMappingError(t *testing.T) { } func TestDesktopResizeAndMovePointer(t *testing.T) { + if os.Getenv("GW_SKIP_PYTHON_TESTS") != "" { + t.Skip("Skipping since GW_SKIP_PYTHON_TESTS env var is set") + } defer setup(t)() if config.RunTasksAsCurrentUser { t.Skip("Skipping since running as current user...") diff --git a/taskcluster_proxy_broken_on_docker_test.go b/taskcluster_proxy_broken_on_docker_test.go index 7a99209e..530e755d 100644 --- a/taskcluster_proxy_broken_on_docker_test.go +++ b/taskcluster_proxy_broken_on_docker_test.go @@ -6,13 +6,13 @@ import ( "fmt" "os" "testing" - - "github.com/taskcluster/generic-worker/testutil" ) func TestTaskclusterProxy(t *testing.T) { - testutil.RequireTaskclusterCredentials(t) defer setup(t)() + + taskID := CreateArtifactFromFile(t, "SampleArtifacts/_/X.txt", "SampleArtifacts/_/X.txt") + payload := GenericWorkerPayload{ Command: append( append( @@ -23,7 +23,7 @@ func TestTaskclusterProxy(t *testing.T) { goRun( "curlget.go", // note that curlget.go supports substituting the proxy URL from its runtime environment - fmt.Sprintf("TASKCLUSTER_PROXY_URL/queue/v1/task/KTBKfEgxR5GdfIIREQIvFQ/runs/0/artifacts/SampleArtifacts/_/X.txt"), + fmt.Sprintf("TASKCLUSTER_PROXY_URL/queue/v1/task/"+taskID+"/runs/0/artifacts/SampleArtifacts/_/X.txt"), )..., ), MaxRunTime: 60, @@ -43,8 +43,9 @@ func TestTaskclusterProxy(t *testing.T) { } td := testTask(t) td.Scopes = []string{"queue:get-artifact:SampleArtifacts/_/X.txt"} + td.Dependencies = []string{taskID} reclaimEvery5Seconds = true - taskID := submitAndAssert(t, td, payload, "completed", "completed") + taskID = submitAndAssert(t, td, payload, "completed", "completed") reclaimEvery5Seconds = false expectedArtifacts := ExpectedArtifacts{ diff --git a/tcproxy/tcproxy_test.go b/tcproxy/tcproxy_test.go index 1a48322d..5e472b36 100644 --- a/tcproxy/tcproxy_test.go +++ b/tcproxy/tcproxy_test.go @@ -1,6 +1,7 @@ package tcproxy import ( + "encoding/json" "io/ioutil" "net/http" "os" @@ -9,6 +10,7 @@ import ( "github.com/taskcluster/generic-worker/testutil" tcclient "github.com/taskcluster/taskcluster-client-go" + "github.com/taskcluster/taskcluster-client-go/tcauth" ) func TestTcProxy(t *testing.T) { @@ -38,7 +40,7 @@ func TestTcProxy(t *testing.T) { if err != nil { t.Fatalf("Could not initiate taskcluster-proxy process:\n%s", err) } - res, err := http.Get("http://localhost:34569/queue/v1/task/KTBKfEgxR5GdfIIREQIvFQ/runs/0/artifacts/SampleArtifacts/_/X.txt") + res, err := http.Get("http://localhost:34569/auth/v1/scopes/current") if err != nil { t.Fatalf("Could not hit url to download artifact using taskcluster-proxy: %v", err) } @@ -47,7 +49,12 @@ func TestTcProxy(t *testing.T) { if err != nil { t.Fatalf("Could not read artifact using taskcluster-proxy: %v", err) } - if string(data) != "test artifact\n" { + scopeset := new(tcauth.SetOfScopes) + err = json.Unmarshal(data, scopeset) + if err != nil { + t.Fatalf("Could not interpret response %q as json: %v", string(data), err) + } + if len(scopeset.Scopes) != 1 || scopeset.Scopes[0] != "queue:get-artifact:SampleArtifacts/_/X.txt" { t.Fatalf("Got incorrect data: %v", string(data)) } } diff --git a/testdata/mozharness.zip b/testdata/mozharness.zip new file mode 100644 index 00000000..42d2421c Binary files /dev/null and b/testdata/mozharness.zip differ diff --git a/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json b/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json deleted file mode 100644 index 69c96896..00000000 --- a/testdata/tasks/KTBKfEgxR5GdfIIREQIvFQ.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "provisionerId": "test-provisioner", - "workerType": "Ga6sJQoGRz225-RGnvbpgQ", - "schedulerId": "test-scheduler", - "taskGroupId": "OY0sHU3VTrykGZLTGZ_goQ", - "dependencies": [], - "requires": "all-completed", - "routes": [], - "priority": "lowest", - "retries": 1, - "created": "2016-09-19T12:09:54.000Z", - "deadline": "2016-09-20T12:09:54.000Z", - "expires": "2021-09-19T12:09:54.000Z", - "scopes": [], - "payload": { - "command": [ - [ - "echo", - "hello world!" - ], - [ - "echo", - "goodbye world!" - ] - ], - "maxRunTime": 7200, - "artifacts": [ - { - "path": "SampleArtifacts/_/X.txt", - "expires": "2021-09-19T12:09:54.000Z", - "type": "file" - } - ], - "features": { - "chainOfTrust": true - } - }, - "metadata": { - "description": "Test task", - "name": "[TC] TestUpload", - "owner": "pmoore@mozilla.com", - "source": "https://github.com/taskcluster/generic-worker/blob/master/artifacts_test.go" - }, - "tags": { - "createdForUser": "pmoore@mozilla.com" - }, - "extra": {} -} \ No newline at end of file diff --git a/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json b/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json deleted file mode 100644 index 0fda0a8e..00000000 --- a/testdata/tasks/LK1Rz2UtT16d-HBSqyCtuA.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "provisionerId": "aws-provisioner-v1", - "workerType": "win2012r2-cu", - "schedulerId": "-", - "taskGroupId": "LK1Rz2UtT16d-HBSqyCtuA", - "dependencies": [], - "requires": "all-completed", - "routes": [], - "priority": "lowest", - "retries": 1, - "created": "2017-09-19T10:44:02.999Z", - "deadline": "2017-09-20T10:44:02.999Z", - "expires": "2030-09-20T10:44:02.999Z", - "scopes": [], - "payload": { - "command": [ - "echo hello" - ], - "maxRunTime": 3600, - "artifacts": [ - { - "type": "file", - "name": "public/build/unknown_issuer_app_1.zip", - "path": "unknown_issuer_app_1.zip" - } - ], - "mounts": [ - { - "file": "unknown_issuer_app_1.zip", - "content": { - "url": "https://hg.mozilla.org/mozilla-central/raw-file/5fc5a88872adb1ca891b85bcdd072b8159a26c05/security/manager/ssl/tests/unit/test_signed_apps/unknown_issuer_app_1.zip" - } - } - ] - }, - "metadata": { - "name": "TestMounts Artifact Generator", - "description": "This task is used by generic-worker CI", - "owner": "pmoore@mozilla.com", - "source": "https://github.com/taskcluster/generic-worker/blob/fa39e57e18fb2350ee855dfad9fa591e0c10a966/mounts_test.go#L95" - }, - "tags": {}, - "extra": {} -} \ No newline at end of file diff --git a/testdata/tasks/README.md b/testdata/tasks/README.md deleted file mode 100644 index 0520fd92..00000000 --- a/testdata/tasks/README.md +++ /dev/null @@ -1 +0,0 @@ -This directory contains the task definitions of tasks that are used in go tests. diff --git a/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json b/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json deleted file mode 100644 index ba23e9ea..00000000 --- a/testdata/tasks/VESwp9JaRo-XkFN_bemBhw.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "provisionerId": "aws-provisioner-v1", - "workerType": "win2012r2-cu", - "schedulerId": "-", - "taskGroupId": "VESwp9JaRo-XkFN_bemBhw", - "dependencies": [], - "requires": "all-completed", - "routes": [], - "priority": "lowest", - "retries": 1, - "created": "2017-09-19T11:12:23.203Z", - "deadline": "2017-09-20T11:12:23.203Z", - "expires": "2030-09-20T11:12:23.203Z", - "scopes": [], - "payload": { - "command": [ - "echo hello" - ], - "maxRunTime": 3600, - "artifacts": [ - { - "type": "file", - "name": "public/build/mozharness.zip", - "path": "mozharness.zip" - } - ], - "mounts": [ - { - "file": "mozharness.zip", - "content": { - "url": "https://github.com/taskcluster/testrepo/blob/b7a05b4e016033b00d5ed15b470451dd7aa61495/generic-worker/mozharness.zip?raw=true" - } - } - ] - }, - "metadata": { - "name": "TestMounts Artifact Generator", - "description": "This task is used by generic-worker CI", - "owner": "pmoore@mozilla.com", - "source": "https://github.com/taskcluster/generic-worker/blob/fa39e57e18fb2350ee855dfad9fa591e0c10a966/mounts_test.go#L95" - }, - "tags": {}, - "extra": {} -} \ No newline at end of file diff --git a/testdata/unknown_issuer_app_1.zip b/testdata/unknown_issuer_app_1.zip new file mode 100644 index 00000000..374e45f6 Binary files /dev/null and b/testdata/unknown_issuer_app_1.zip differ diff --git a/vendor/github.com/taskcluster/shell/shell.go b/vendor/github.com/taskcluster/shell/shell.go index a16121aa..aa1f8ff2 100644 --- a/vendor/github.com/taskcluster/shell/shell.go +++ b/vendor/github.com/taskcluster/shell/shell.go @@ -9,6 +9,10 @@ func Escape(tokens ...string) (escaped string) { var escapedToken string escapedTokens := []string{} for _, j := range tokens { + if j == "" { + escapedTokens = append(escapedTokens, `''`) + continue + } for _, k := range j { if !strings.ContainsRune(safeChars, k) { goto escape