From fcac32a8129307c7b00b9720bdcaf828dcd641fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Ch=C3=A1varri?= Date: Wed, 11 Dec 2019 03:39:27 +0100 Subject: [PATCH] Upgrade ci to use latest from hello-reason (#361) * Upgrade ci to use latest from hello-reason * esy-build-steps->build-platform * Add publish-build-cache template --- .ci/build-platform.yml | 37 ++++++++++ .ci/esy-build-steps.yml | 22 ------ .ci/publish-build-cache.yml | 25 ------- .ci/restore-build-cache.yml | 40 ----------- .ci/use-calc-esy-install-path.yml | 32 --------- .ci/utils/publish-build-cache.yml | 3 + .ci/utils/restore-build-cache.yml | 45 ++++++++++++ .ci/utils/use-esy.yml | 5 ++ .ci/utils/use-node.yml | 7 ++ azure-pipelines.yml | 112 +++++++++++------------------- 10 files changed, 138 insertions(+), 190 deletions(-) create mode 100644 .ci/build-platform.yml delete mode 100644 .ci/esy-build-steps.yml delete mode 100644 .ci/publish-build-cache.yml delete mode 100644 .ci/restore-build-cache.yml delete mode 100644 .ci/use-calc-esy-install-path.yml create mode 100644 .ci/utils/publish-build-cache.yml create mode 100644 .ci/utils/restore-build-cache.yml create mode 100644 .ci/utils/use-esy.yml create mode 100644 .ci/utils/use-node.yml diff --git a/.ci/build-platform.yml b/.ci/build-platform.yml new file mode 100644 index 00000000..cb7019b0 --- /dev/null +++ b/.ci/build-platform.yml @@ -0,0 +1,37 @@ +# Cross-platform set of build steps for building esy projects + +parameters: + platform: "macOS" + vmImage: "macOS-10.13" + +jobs: + - job: ${{ parameters.platform }} + pool: + vmImage: ${{ parameters.vmImage }} + demands: node.js + timeoutInMinutes: 120 # This is mostly for Windows + variables: + ESY__CACHE_INSTALL_PATH: ${{ parameters.ESY__CACHE_INSTALL_PATH }} + CACHE_FOLDER: $(Pipeline.Workspace)/cache + steps: + - template: utils/use-node.yml + - template: utils/use-esy.yml + - template: utils/restore-build-cache.yml + - script: "esy install" + displayName: "esy install" + - script: "esy build" + displayName: "esy build" + # - script: "esy test" + # displayName: "Test command" + - template: utils/publish-build-cache.yml + - bash: ./_build/default/src/analyze_example_tests/ExamplesTests.exe + - bash: ./_build/default/util_tests/UtilTests.exe + - bash: mkdir -p rls-release + - bash: cp _build/default/bin/Bin.exe rls-release/reason-language-server.exe + - task: PublishBuildArtifacts@1 + displayName: 'Upload binary' + inputs: + pathToPublish: 'rls-release' + artifactName: 'rls-$(Agent.OS)' + parallel: true + parallelCount: 8 diff --git a/.ci/esy-build-steps.yml b/.ci/esy-build-steps.yml deleted file mode 100644 index 3fb1d9ab..00000000 --- a/.ci/esy-build-steps.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Cross-platform set of build steps for building esy projects - -steps: - - script: npm install -g esy@0.5.6 - displayName: 'npm install -g esy@0.5.6' - - script: esy install - displayName: 'esy install' - - script: esy build - displayName: 'esy build' - # Run tests or any additional steps here - # - script: esy b dune runtest - - bash: ./_build/default/src/analyze_example_tests/ExamplesTests.exe - - bash: ./_build/default/util_tests/UtilTests.exe - - bash: mkdir -p rls-release - - bash: cp _build/default/bin/Bin.exe rls-release/reason-language-server.exe - - task: PublishBuildArtifacts@1 - displayName: 'Upload binary' - inputs: - pathToPublish: 'rls-release' - artifactName: 'rls-$(Agent.OS)' - parallel: true - parallelCount: 8 diff --git a/.ci/publish-build-cache.yml b/.ci/publish-build-cache.yml deleted file mode 100644 index e3bad226..00000000 --- a/.ci/publish-build-cache.yml +++ /dev/null @@ -1,25 +0,0 @@ -# Steps for publishing project cache - -steps: - - bash: 'mkdir -p $(STAGING_DIRECTORY_UNIX)' - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - displayName: '[Cache][Publish] Create cache directory' - - - bash: 'cd $(ESY__CACHE_INSTALL_PATH) && tar -czf $(STAGING_DIRECTORY_UNIX)/esy-cache.tar .' - workingDirectory: '' - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - displayName: '[Cache][Publish] Tar esy cache directory' - - # - bash: 'cd $(ESY__NPM_ROOT) && tar -czf $(STAGING_DIRECTORY_UNIX)/npm-cache.tar .' - # condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - # displayName: '[Cache][Publish] Tar npm cache directory' - - - task: PublishBuildArtifacts@1 - displayName: '[Cache][Publish] Upload tarball' - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - inputs: - pathToPublish: '$(STAGING_DIRECTORY)' - artifactName: 'cache-$(Agent.OS)-install' - parallel: true - parallelCount: 8 - \ No newline at end of file diff --git a/.ci/restore-build-cache.yml b/.ci/restore-build-cache.yml deleted file mode 100644 index ed82091b..00000000 --- a/.ci/restore-build-cache.yml +++ /dev/null @@ -1,40 +0,0 @@ -# Steps for restoring project cache - -steps: - - task: DownloadBuildArtifacts@0 - condition: succeeded() - # condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/master')) - displayName: '[Cache][Restore] Restore install' - inputs: - buildType: 'specific' - project: '$(System.TeamProject)' - pipeline: '$(Build.DefinitionName)' - branchName: 'refs/heads/master' - buildVersionToDownload: 'latestFromBranch' - downloadType: 'single' - artifactName: 'cache-$(Agent.OS)-install' - downloadPath: '$(STAGING_DIRECTORY)' - continueOnError: true - - - bash: 'mkdir -p $(ESY__CACHE_INSTALL_PATH)' - condition: succeeded() - # condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/master')) - displayName: '[Cache][Restore] Create cache directory' - - # - bash: 'cd $(ESY__NPM_ROOT) && tar -xf $(STAGING_DIRECTORY_UNIX)/cache-$(Agent.OS)-install/npm-cache.tar -C .' - # continueOnError: true - # condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/master')) - # displayName: '[Cache][Restore] Untar npm cache directory' - - - bash: 'cd $(ESY__CACHE_INSTALL_PATH) && tar -xf $(STAGING_DIRECTORY_UNIX)/cache-$(Agent.OS)-install/esy-cache.tar -C .' - continueOnError: true - condition: succeeded() - # condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/master')) - displayName: '[Cache][Restore] Untar esy cache directory' - - - bash: 'rm -rf *' - continueOnError: true - workingDirectory: '$(STAGING_DIRECTORY)' - condition: succeeded() - # condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/master')) - displayName: '[Cache][Restore] Clean up' diff --git a/.ci/use-calc-esy-install-path.yml b/.ci/use-calc-esy-install-path.yml deleted file mode 100644 index 6cf7df22..00000000 --- a/.ci/use-calc-esy-install-path.yml +++ /dev/null @@ -1,32 +0,0 @@ -steps: - - bash: | - # COMPUTE THE ESY INSTALL CACHE LOCATION AHEAD OF TIME - DESIRED_LEN="86" - # Note: This will need to change when upgrading esy version - # that reenables long paths on windows. - if [ "$AGENT_OS" == "Windows_NT" ]; then - DESIRED_LEN="33" - fi - HOME_ESY3="$HOME/.esy/3" - HOME_ESY3_LEN=${#HOME_ESY3} - NUM_UNDERS=$(echo "$(($DESIRED_LEN-$HOME_ESY3_LEN))") - UNDERS=$(printf "%-${NUM_UNDERS}s" "_") - UNDERS="${UNDERS// /_}" - ESY__CACHE_INSTALL_PATH=${HOME_ESY3}${UNDERS}/i - if [ "$AGENT_OS" == "Windows_NT" ]; then - ESY__CACHE_INSTALL_PATH=$( cygpath --mixed --absolute "$ESY__CACHE_INSTALL_PATH") - fi - echo "ESY__CACHE_INSTALL_PATH: $ESY__CACHE_INSTALL_PATH" - # This will be exposed as an env var ESY__CACHE_INSTALL_PATH, or an - # Azure var esy__cache_install_path - echo "##vso[task.setvariable variable=esy__cache_install_path]$ESY__CACHE_INSTALL_PATH" - displayName: "Task.setvariable ESY__CACHE_INSTALL_PATH" - # - bash: | - # which esy - # echo "$( which esy )" - # echo "##vso[task.setvariable variable=esy_bin_location]$(which esy)" - # displayName: "Find esy binary" - # - bash: echo ${ESY_BIN_LOCATION} - # displayName: "Print esy bin location" - - bash: env - displayName: "Print environment" diff --git a/.ci/utils/publish-build-cache.yml b/.ci/utils/publish-build-cache.yml new file mode 100644 index 00000000..fcd8502b --- /dev/null +++ b/.ci/utils/publish-build-cache.yml @@ -0,0 +1,3 @@ +steps: + - pwsh: Copy-Item -Path $(ESY__CACHE_INSTALL_PATH) -Destination $(CACHE_FOLDER) -Recurse + displayName: '[Cache][Publish] Copy builds to be cached' diff --git a/.ci/utils/restore-build-cache.yml b/.ci/utils/restore-build-cache.yml new file mode 100644 index 00000000..fef0a953 --- /dev/null +++ b/.ci/utils/restore-build-cache.yml @@ -0,0 +1,45 @@ +# The cache key is built up of the following: +# We use a string that we can change to bust the cache +# The string "esy" +# The string for the OS +# The hash of the lock file +steps: + - bash: | + # COMPUTE THE ESY INSTALL CACHE LOCATION AHEAD OF TIME + DESIRED_LEN="86" + # Note: This will need to change when upgrading esy version + # that reenables long paths on windows. + if [ "$AGENT_OS" == "Windows_NT" ]; then + DESIRED_LEN="33" + fi + HOME_ESY3="$HOME/.esy/3" + HOME_ESY3_LEN=${#HOME_ESY3} + NUM_UNDERS=$(echo "$(($DESIRED_LEN-$HOME_ESY3_LEN))") + UNDERS=$(printf "%-${NUM_UNDERS}s" "_") + UNDERS="${UNDERS// /_}" + THE_ESY__CACHE_INSTALL_PATH=${HOME_ESY3}${UNDERS}/i + if [ "$AGENT_OS" == "Windows_NT" ]; then + THE_ESY__CACHE_INSTALL_PATH=$( cygpath --mixed --absolute "$THE_ESY__CACHE_INSTALL_PATH") + fi + echo "THE_ESY__CACHE_INSTALL_PATH: $THE_ESY__CACHE_INSTALL_PATH" + # This will be exposed as an env var ESY__CACHE_INSTALL_PATH, or an + # Azure var esy__cache_install_path + echo "##vso[task.setvariable variable=esy__cache_install_path]$THE_ESY__CACHE_INSTALL_PATH" + displayName: '[Cache] calculate esy store path' + + - task: Cache@2 + inputs: + # "v1" prefix added just to keep the ability to clear a cache without having to wait 7 days + key: v1 | esy | $(Agent.OS) | esy.lock/index.json + restoreKeys: v1 | esy | $(Agent.OS) + path: $(CACHE_FOLDER) + cacheHitVar: CACHE_RESTORED + displayName: '[Cache] esy packages' + + - pwsh: 'New-Item $(ESY__CACHE_INSTALL_PATH) -ItemType Directory' + condition: eq(variables.CACHE_RESTORED, 'true') + displayName: '[Cache][Restore] Create esy cache directory' + + - pwsh: Move-Item -Path $(CACHE_FOLDER)/* -Destination $(ESY__CACHE_INSTALL_PATH) + displayName: '[Cache][Restore] Move downloaded cache in place' + condition: eq(variables.CACHE_RESTORED, 'true') diff --git a/.ci/utils/use-esy.yml b/.ci/utils/use-esy.yml new file mode 100644 index 00000000..cf9ab99e --- /dev/null +++ b/.ci/utils/use-esy.yml @@ -0,0 +1,5 @@ +# steps to install esy globally + +steps: + - script: "npm install -g esy@0.5.8" + displayName: "install esy" diff --git a/.ci/utils/use-node.yml b/.ci/utils/use-node.yml new file mode 100644 index 00000000..86ac8daf --- /dev/null +++ b/.ci/utils/use-node.yml @@ -0,0 +1,7 @@ +# steps to use node on agent + +steps: + - task: NodeTool@0 + displayName: "Use Node 8.x" + inputs: + versionSpec: 8.x diff --git a/azure-pipelines.yml b/azure-pipelines.yml index eacd842b..df1b9b19 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,81 +1,51 @@ -# Starter pipeline -# Start with a minimal pipeline that you can customize to build and deploy your code. -# Add steps that build, run tests, deploy, and more: -# https://aka.ms/yaml - - name: $(Build.SourceVersion) -jobs: -- job: Linux - timeoutInMinutes: 0 - pool: - vmImage: 'Ubuntu 16.04' - - variables: - STAGING_DIRECTORY: /home/vsts/STAGING - STAGING_DIRECTORY_UNIX: /home/vsts/STAGING - ESY__CACHE_INSTALL_PATH: /home/vsts/.esy/3_____________________________________________________________________/i - ESY__CACHE_SOURCE_TARBALL_PATH: /home/vsts/.esy/source/i - # ESY__NPM_ROOT: /opt/hostedtoolcache/node/8.14.0/x64/lib/node_modules/esy - steps: - - template: .ci/use-node.yml - - template: .ci/restore-build-cache.yml - - template: .ci/esy-build-steps.yml - - template: .ci/publish-build-cache.yml +trigger: + branches: + include: + - master -- job: MacOS - timeoutInMinutes: 0 - pool: - vmImage: 'macOS 10.13' +pr: + branches: + include: + - "*" - variables: - STAGING_DIRECTORY: /Users/runner/STAGING - STAGING_DIRECTORY_UNIX: /Users/runner/STAGING - ESY__CACHE_SOURCE_TARBALL_PATH: /Users/runner/.esy/source/i - # ESY__NPM_ROOT: /usr/local/lib/node_modules/esy - - steps: - - template: .ci/use-calc-esy-install-path.yml - - template: .ci/use-node.yml - - template: .ci/restore-build-cache.yml - - template: .ci/esy-build-steps.yml - - template: .ci/publish-build-cache.yml - -- job: Windows - timeoutInMinutes: 0 - pool: - vmImage: 'vs2017-win2016' +jobs: + - template: .ci/build-platform.yml + parameters: + platform: Linux + vmImage: ubuntu-16.04 - variables: - STAGING_DIRECTORY: C:\Users\VssAdministrator\STAGING - STAGING_DIRECTORY_UNIX: /C/Users/VssAdministrator/STAGING - ESY__CACHE_INSTALL_PATH: /C/Users/VssAdministrator/.esy/3_/i - ESY__CACHE_SOURCE_TARBALL_PATH: /C/Users/VssAdministrator/.esy/source/i - # ESY__NPM_ROOT: /C/npm/prefix/node_modules/esy + - template: .ci/build-platform.yml + parameters: + platform: macOS + vmImage: macOS-10.13 - steps: - - template: .ci/use-node.yml - - template: .ci/restore-build-cache.yml - - template: .ci/esy-build-steps.yml - - template: .ci/publish-build-cache.yml + # Need windows-2019 to do esy import/export-dependencies + # which assumes you have bsdtar (tar.exe) in your system + # otherwise it will end up using the esy-bash tar which doesn't + # understand drives like D:/ (thinks it's an scp path). + - template: .ci/build-platform.yml + parameters: + platform: Windows + vmImage: windows-2019 -- job: Release - timeoutInMinutes: 0 - displayName: Release - dependsOn: - - Linux - - MacOS - - Windows - condition: succeeded() - pool: - vmImage: ubuntu-16.04 - steps: - - task: PublishBuildArtifacts@1 - displayName: 'Release Package' - inputs: - PathtoPublish: './editor-extensions/vscode/' - ArtifactName: npm-package + - job: Release + timeoutInMinutes: 0 + displayName: Release + dependsOn: + - Linux + - MacOS + - Windows + condition: succeeded() + pool: + vmImage: ubuntu-16.04 + steps: + - task: PublishBuildArtifacts@1 + displayName: 'Release Package' + inputs: + PathtoPublish: './editor-extensions/vscode/' + ArtifactName: npm-package # - name: release