Refactor pipelines into stages

.vscode/settings.json

@@ -38,7 +38,7 @@
     "python.jediEnabled": false,
     "python.analysis.logLevel": "Trace",
     "python.analysis.downloadChannel": "beta",
-   "python.linting.pylintEnabled": false,
+    "python.linting.pylintEnabled": false,
     "python.linting.flake8Enabled": true,
     "cucumberautocomplete.skipDocStringsFormat": true,
     "python.linting.flake8Args": [

build/ci/templates/globals.yml

@@ -0,0 +1,8 @@
+variables:
+  PythonVersion: '3.7.4' # Always use latest version.
+  NodeVersion: '10.11.0' # Check version of node used in VS Code.
+  NpmVersion: '6.10.3'
+  MOCHA_FILE: '$(Build.ArtifactStagingDirectory)/test-junit.xml' # All test files will write their JUnit xml output to this file, clobbering the last time it was written.
+  MOCHA_REPORTER_JUNIT: true # Use the mocha-multi-reporters and send output to both console (spec) and JUnit (mocha-junit-reporter).
+  VSC_PYTHON_FORCE_LOGGING: true # Enable this to turn on console output for the logger
+  VSC_PYTHON_LOG_FILE: '$(Build.ArtifactStagingDirectory)/pvsc.log'

build/ci/templates/build_compile_jobs.yml → build/ci/templates/jobs/build_compile.yml

@@ -3,17 +3,19 @@
 
 jobs:
 - job: Compile
-  variables:
-    build: false
   pool:
     vmImage: "macos-latest"
   steps:
-    - template: build_compile_steps.yml
+    - template: ../steps/compile.yml
 
 - job: Build
-  variables:
-    build: true
   pool:
     vmImage: "macos-latest"
   steps:
-    - template: build_compile_steps.yml
+    - template: ../steps/build.yml
+
+- job: Lint
+  pool:
+    vmImage: "macos-latest"
+  steps:
+    - template: ../steps/lint.yml

build/ci/templates/jobs/coverage.yml

@@ -0,0 +1,8 @@
+jobs:
+- job: Coverage
+  pool:
+    vmImage: "macos-latest"
+  variables:
+    TestsToRun: 'testUnitTests'
+  steps:
+    - template: ../steps/merge_upload_coverage.yml

build/ci/templates/jobs/smoke_test.yml

@@ -0,0 +1,10 @@
+jobs:
+- template: uitest.yml
+  parameters:
+    # In PRs, test only against stable version of VSC.
+    vscodeChannels: ['stable']
+    # In PRs, run smoke tests against 3.7 and 2.7 (excluding others).
+    jobs:
+    - test: "Smoke"
+      tags: "@smoke"
+      ignorePythonVersions: "3.6,3.5"

build/ci/templates/steps/build.yml

@@ -0,0 +1,63 @@
+# -----------------------------------------------------------------------------------------------------------------------------
+# Overview:
+# -----------------------------------------------------------------------------------------------------------------------------
+# Set of steps used to compile and build the extension
+#
+# -----------------------------------------------------------------------------------------------------------------------------
+# Variables
+# -----------------------------------------------------------------------------------------------------------------------------
+# 1. build
+#   Mandatory
+#   Possible values, `true` or `false`.
+#   If `true`, means we need to build the VSIX, else just compile.
+
+steps:
+    - template: initialization.yml
+      parameters:
+        PythonVersion: $(PythonVersion)
+        workingDirectory: $(Build.SourcesDirectory)
+        compile: 'false'
+
+    - bash: |
+          python -m pip install -U pip
+          python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt
+          python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python/old_ptvsd --no-cache-dir --implementation py --no-deps --upgrade 'ptvsd==4.3.2'
+      failOnStderr: true
+      displayName: "pip install requirements"
+
+    - task: PythonScript@0
+      displayName: "Install PTVSD wheels"
+      inputs:
+          scriptSource: "filePath"
+          scriptPath: "./pythonFiles/install_ptvsd.py"
+          arguments: "--ci"
+          failOnStderr: true
+
+    - bash: npm run clean
+      displayName: "Clean"
+
+    - bash: |
+          npm run updateBuildNumber -- --buildNumber $BUILD_BUILDID
+      displayName: "Update dev Version of Extension"
+      condition: and(succeeded(), eq(variables['Build.SourceBranchName'], 'master'))
+
+    - bash: |
+          npm run updateBuildNumber -- --buildNumber $BUILD_BUILDID --updateChangelog
+      displayName: "Update release Version of Extension"
+      condition: and(succeeded(), eq(variables['Build.SourceBranchName'], 'release'))
+
+    - bash: |
+          npm run package
+      displayName: "Build VSIX"
+
+    - task: CopyFiles@2
+      inputs:
+          contents: "*.vsix"
+          targetFolder: $(Build.ArtifactStagingDirectory)
+      displayName: "Copy VSIX"
+
+    - task: PublishBuildArtifacts@1
+      inputs:
+          pathtoPublish: $(Build.ArtifactStagingDirectory)
+          artifactName: VSIX
+      displayName: "Publish VSIX to Artifacts"

build/ci/templates/build_compile_steps.yml → build/ci/templates/steps/build_compile.yml

@@ -12,43 +12,11 @@
 #   If `true`, means we need to build the VSIX, else just compile.
 
 steps:
-    - bash: |
-          printenv
-      displayName: "Show all env vars"
-      condition: eq(variables['system.debug'], 'true')
-
-    - task: NodeTool@0
-      displayName: "Use Node $(NodeVersion)"
-      inputs:
-          versionSpec: $(NodeVersion)
-
-    - task: UsePythonVersion@0
-      displayName: "Use Python $(PythonVersion)"
-      inputs:
-          versionSpec: $(PythonVersion)
-
-    - task: Npm@1
-      displayName: "Use NPM $(NpmVersion)"
-      inputs:
-          command: custom
-          verbose: true
-          customCommand: "install -g npm@$(NpmVersion)"
-
-    - task: Npm@1
-      displayName: "npm ci"
-      inputs:
-          command: custom
-          verbose: true
-          customCommand: ci
-
-    - bash: |
-          echo AVAILABLE DEPENDENCY VERSIONS
-          echo Node Version = `node -v`
-          echo NPM Version = `npm -v`
-          echo Python Version = `python --version`
-          echo Gulp Version = `gulp --version`
-      condition: and(succeeded(), eq(variables['system.debug'], 'true'))
-      displayName: Show Dependency Versions
+    - template: initialization.yml
+      parameters:
+        PythonVersion: $(PythonVersion)
+        workingDirectory: $(Build.SourcesDirectory)
+        compile: 'false'
 
     - task: Gulp@0
       displayName: "Compile and check for errors"
@@ -77,20 +45,18 @@ steps:
           failOnStderr: true
       condition: and(succeeded(), eq(variables['build'], 'true'))
 
-    - bash: |
-          npm install -g vsce
-          npm run clean
-      displayName: "Install vsce & Clean"
+    - bash: npm run clean
+      displayName: "Clean"
       condition: and(succeeded(), eq(variables['build'], 'true'))
 
     - bash: |
           npm run updateBuildNumber -- --buildNumber $BUILD_BUILDID
-      displayName: "Update dev Version"
+      displayName: "Update dev Version of Extension"
       condition: and(succeeded(), eq(variables['build'], 'true'), eq(variables['Build.SourceBranchName'], 'master'))
 
     - bash: |
           npm run updateBuildNumber -- --buildNumber $BUILD_BUILDID --updateChangelog
-      displayName: "Update release Version"
+      displayName: "Update release Version of Extension"
       condition: and(succeeded(), eq(variables['build'], 'true'), eq(variables['Build.SourceBranchName'], 'release'))
 
     - bash: |

build/ci/templates/steps/compile.yml

@@ -0,0 +1,14 @@
+# Compiles the source
+
+steps:
+    - template: initialization.yml
+      parameters:
+        PythonVersion: $(PythonVersion)
+        workingDirectory: $(Build.SourcesDirectory)
+        compile: 'false'
+        installVSCEorNPX: 'false'
+
+    - task: Gulp@0
+      displayName: "Compile and check for errors"
+      inputs:
+          targets: "prePublishNonBundle"

build/ci/templates/steps/initialization.yml

@@ -0,0 +1,113 @@
+# -----------------------------------------------------------------------------------------------------------------------------
+# Overview:
+# -----------------------------------------------------------------------------------------------------------------------------
+# Set of common steps required when initializing a job.
+#
+# -----------------------------------------------------------------------------------------------------------------------------
+# Variables
+# -----------------------------------------------------------------------------------------------------------------------------
+# 1. workingDirectory
+#   Mandatory
+#   Working directory.
+# 2. PythonVersion
+#   Python version to be used.
+#   If not provided, python will not be setup on CI.
+parameters:
+  workingDirectory: ''
+  PythonVersion: ''
+  compile: 'true'
+  sqlite: 'false'
+  installVSCEorNPX: 'true'
+
+steps:
+    - bash: |
+          printenv
+      displayName: "Show all env vars"
+      condition: eq(variables['system.debug'], 'true')
+
+    - task: NodeTool@0
+      displayName: "Use Node $(NodeVersion)"
+      inputs:
+          versionSpec: $(NodeVersion)
+
+    - task: UsePythonVersion@0
+      displayName: "Use Python ${{ parameters.PythonVersion }}"
+      condition: and(succeeded(), not(eq('${{ parameters.PythonVersion }}', '')))
+      inputs:
+          versionSpec: ${{ parameters.PythonVersion }}
+
+    # Install the a version of python that works with sqlite3 until this bug is addressed
+    # https://mseng.visualstudio.com/AzureDevOps/_workitems/edit/1535830
+    #
+    # This task will only run if variable `NeedsPythonFunctionalReqs` is true.
+    - bash: |
+        sudo apt-get install libsqlite3-dev
+        version=$(python -V 2>&1 | grep -Po '(?<=Python )(.+)')
+        wget https://www.python.org/ftp/python/$version/Python-$version.tar.xz
+        tar xvf Python-$version.tar.xz
+        cd Python-$version
+        ./configure --enable-loadable-sqlite-extensions --with-ensurepip=install --prefix=$HOME/py-$version
+        make
+        sudo make install
+        sudo chmod -R 777 $HOME/py-$version
+        export PATH=$HOME/py-$version/bin:$PATH
+        sudo ln -s $HOME/py-$version/bin/python3 $HOME/py-$version/bin/python
+        echo '##vso[task.prependpath]'$HOME/py-$version/bin
+      displayName: 'Setup python to run with sqlite on 3.*'
+      condition: and(succeeded(), eq('${{ parameters.sqlite }}', 'true'), not(eq('${{ parameters.PythonVersion }}', '')), not(eq('${{ parameters.PythonVersion }}', '2.7')), eq(variables['Agent.Os'], 'Linux'))
+
+    - task: Npm@1
+      displayName: "Use NPM $(NpmVersion)"
+      inputs:
+          command: custom
+          verbose: true
+          customCommand: "install -g npm@$(NpmVersion)"
+
+    - task: Npm@1
+      displayName: "npm ci"
+      inputs:
+          workingDir: ${{ parameters.workingDirectory }}
+          command: custom
+          verbose: true
+          customCommand: ci
+
+    # On Mac, the command `node` doesn't always point to the current node version.
+    # Debugger tests use the variable process.env.NODE_PATH
+    - script: |
+          export NODE_PATH=`which node`
+          echo $NODE_PATH
+          echo '##vso[task.setvariable variable=NODE_PATH]'$NODE_PATH
+      displayName: "Setup NODE_PATH for extension (Debugger Tests)"
+      condition: and(succeeded(), eq(variables['agent.os'], 'Darwin'))
+
+    # We need these almost always.
+    - bash: |
+          npm install -g vsce
+          npm install -g npx
+      displayName: "Install vsce & npx"
+      condition: and(succeeded(), eq('${{ parameters.installVSCEorNPX }}', 'true'))
+
+    - bash: npx tsc -p ./
+      displayName: "compile (npx tsc -p ./)"
+      workingDirectory: ${{ parameters.workingDirectory }}
+      condition: and(succeeded(), eq('${{ parameters.compile }}', 'true'))
+
+    # https://code.visualstudio.com/api/working-with-extensions/continuous-integration#azure-pipelines
+    - bash: |
+          set -e
+          /usr/bin/Xvfb :10 -ac >> /tmp/Xvfb.out 2>&1 &
+          disown -ar
+      displayName: 'Start Display Server (xvfb) to launch VS Code)'
+      condition: and(succeeded(), eq(variables['Agent.Os'], 'Linux'))
+
+    # # Show all versions installed/available on PATH if in verbose mode.
+    # # Example command line (windows pwsh):
+    # # > Write-Host Node ver: $(& node -v) NPM Ver: $(& npm -v) Python ver: $(& python --version)"
+    # - bash: |
+    #     echo AVAILABLE DEPENDENCY VERSIONS
+    #     echo Node Version = `node -v`
+    #     echo NPM Version = `npm -v`
+    #     echo Python Version = `python --version`
+    #     echo Gulp Version = `gulp --version`
+    #   condition: and(succeeded(), eq(variables['system.debug'], 'true'))
+    #   displayName: Show Dependency Versions

build/ci/templates/steps/lint.yml

@@ -0,0 +1,13 @@
+# Lint the source
+
+steps:
+    - template: initialization.yml
+      parameters:
+        PythonVersion: $(PythonVersion)
+        workingDirectory: $(Build.SourcesDirectory)
+        compile: 'false'
+        installVSCEorNPX: 'false'
+
+    - bash: npx tslint --project tsconfig.json
+      displayName: "Lint"
+      workingDirectory: $(Build.SourcesDirectory)

build/ci/templates/merge_upload_coverage.yml → build/ci/templates/steps/merge_upload_coverage.yml

@@ -1,27 +1,8 @@
 steps:
-    - bash: |
-          printenv
-      displayName: "Show all env vars"
-      condition: eq(variables['system.debug'], 'true')
-
-    - task: NodeTool@0
-      displayName: "Use Node $(NodeVersion)"
-      inputs:
-          versionSpec: $(NodeVersion)
-
-    - task: Npm@1
-      displayName: "Use NPM $(NpmVersion)"
-      inputs:
-          command: custom
-          verbose: true
-          customCommand: "install -g npm@$(NpmVersion)"
-
-    - task: Npm@1
-      displayName: "npm ci"
-      inputs:
-          command: custom
-          verbose: true
-          customCommand: ci
+    - template: initialization.yml
+      parameters:
+        workingDirectory: $(Build.SourcesDirectory)
+        compile: 'false'
 
     - task: DownloadBuildArtifacts@0
       inputs: