Merge pull request #15 from microsoft/laserprec/ci_templates

Add the follow CI pipelines and use template to construct all pipelines:

1. Nightly Build
2. PR Gate with full test matrix for unit and e2e tests

devops/nightly.yml

@@ -0,0 +1,57 @@
+name: $(Date:yyyyMMdd).$(Rev:r)
+
+trigger: none # nightly build is scheduled once per day
+
+variables:
+  - group: azureResourceKeys
+
+stages:
+- stage: static_analysis
+  jobs:
+  - job: flake8_linux_py36
+    pool:
+      vmImage: 'ubuntu-latest'
+    steps:
+    - template: templates/base/run-linter.yml
+      parameters:
+        pyVersion: '3.6'
+
+- stage: unit_tests
+  dependsOn: static_analysis
+  jobs:
+  - template: templates/run-tests-on-multiple-os-py.yml
+    parameters:
+      pyVersions: ['3.6', '3.7', '3.8']
+      testTypes: ['unit', 'io']
+      imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+
+- stage: e2e_tests
+  dependsOn: static_analysis
+  jobs:
+  - template: templates/run-tests-on-multiple-os-py.yml
+    parameters:
+      pyVersions: ['3.6']
+      testTypes: ['azure', 'slow']
+      imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+
+- stage: collect_final_code_coverage
+  dependsOn: 
+  - unit_tests
+  - e2e_tests
+  jobs:
+  - template: templates/merge-cov-reports.yml
+
+- stage: publish_artifacts
+  jobs:
+  - job: archive_wheel_and_sdist
+    pool:
+      vmImage: 'ubuntu-latest'
+    steps:
+    - template: templates/build_wheel_n_sdist.yml
+
+    - task: PublishBuildArtifacts@1
+      inputs:
+        PathtoPublish: $(Build.SourcesDirectory)/dist
+        ArtifactName: distribution_artifacts
+        publishLocation: 'Container'
+      displayName: 'Publish wheel and sdist'

devops/pr-gate.yml

@@ -1,93 +1,45 @@
-trigger:
-  branches:
-    include:
-      - main
-
-strategy:
-  matrix:
-    linux_x64_py3.6:
-      imageName: 'ubuntu-18.04'
-      python.version: '3.6'
-
-    # windows_x64_py3.6:
-    #   imageName: 'windows-2019'
-    #   python.version: '3.6'
-
-    # macos_x64_py3.6:
-    #   imageName: 'macOS-10.14'
-    #   python.version: '3.6'
-
-    # To build using more python versions follow the syntax below:
-    # linux_x64_py3.5:
-    #   imageName: 'ubuntu-16.04'
-    #   python.version: '3.5'
-
-pool:
-  vmImage: '$(imageName)'
-
 name: $(Date:yyyyMMdd).$(Rev:r)
-steps:
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: '$(python.version)'
-    addToPath: true
-    architecture: 'x64'
-  displayName: 'Use Python $(python.version)'
-
-- bash: |
-    python -m pip install --upgrade pip 
-    python -m pip install setuptools wheel
-    python -m pip install -r requirements.txt 
-    python -m pip install -r requirements-dev.txt
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Install dependencies'
-
-- bash: |
-    tox -e flake8
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Run Linter (flake8)'
-
-- bash: |
-    tox -e py
-  env:
-    BLOB_KEY : $(BLOB_KEY)
-    SEARCH_SERVICE_KEY: $(SEARCH_SERVICE_KEY)
-    COGNITIVE_SERVICE_KEY: $(COGNITIVE_SERVICE_KEY)
-    COMPUTER_VISION_SUBSCRIPTION_KEY: $(COMPUTER_VISION_SUBSCRIPTION_KEY) 
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Running unit tests'
-
-
-- task: PublishTestResults@2
-  inputs:
-    testResultsFormat: 'JUnit' 
-    testResultsFiles: 'junit/*.xml' 
-    searchFolder: $(Build.SourcesDirectory)
-    testRunTitle: $(imageName) Build
-    buildPlatform: $(imageName)
-  displayName: 'Publish unit test report'
-
-- task: PublishCodeCoverageResults@1
-  inputs:
-    codeCoverageTool: Cobertura
-    summaryFileLocation: $(Build.SourcesDirectory)/coverage.xml
-    reportDirectory: $(Build.SourcesDirectory)/htmlcov
-  displayName: 'Publish test coverage'
-
-- bash: |
-    python setup.py bdist_wheel --build-number $(Build.BuildNumber) --dist-dir dist
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Building wheel package'
-
-- bash: |
-    mkdir $BUILD_ARTIFACTSTAGINGDIRECTORY/py$(python.version)/
-    cp -r dist/* $BUILD_ARTIFACTSTAGINGDIRECTORY/py$(python.version)/
-  workingDirectory: $(Build.SourcesDirectory)
-  displayName: 'Copying builds to artifact staging dir'
 
-- task: PublishBuildArtifacts@1
-  inputs:
-    PathtoPublish: $(Build.ArtifactStagingDirectory)
-    ArtifactName: genalog
-    publishLocation: 'Container'
-  displayName: 'Publish build as artifacts'
+trigger: none # trigger only via pr
+
+pr:
+- main
+
+variables:
+- group: azureResourceKeys
+
+stages:
+- stage: static_analysis
+  jobs:
+  - job: flake8_linux_py36
+    pool:
+      vmImage: 'ubuntu-latest'
+    steps:
+      - template: templates/base/run-linter.yml
+        parameters:
+          pyVersion: '3.6'
+
+- stage: unit_tests
+  dependsOn: static_analysis
+  jobs:
+  - template: templates/run-tests-on-multiple-os-py.yml
+    parameters:
+      pyVersions: ['3.6', '3.7', '3.8']
+      testTypes: ['unit', 'io']
+      imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+
+- stage: e2e_tests
+  dependsOn: static_analysis
+  jobs:
+  - template: templates/run-tests-on-multiple-os-py.yml
+    parameters:
+      pyVersions: ['3.6']
+      testTypes: ['azure', 'slow']
+      imageOSs: ['ubuntu-18.04']  # 'windows-latest', 'macos-latest'
+
+- stage: collect_final_code_coverage
+  dependsOn: 
+  - unit_tests
+  - e2e_tests
+  jobs:
+  - template: templates/merge-cov-reports.yml

devops/templates/base/publish-test-results.yml

@@ -0,0 +1,15 @@
+# Template for publishing test result report
+parameters:
+- name: pyVersion
+  type: string
+
+steps:
+- task: PublishTestResults@2
+  inputs:
+    testResultsFormat: 'JUnit' 
+    testResultsFiles: 'junit/*.xml' 
+    searchFolder: $(Build.SourcesDirectory)
+    testRunTitle: $(Agent.OS) py$(pyVersion) Build
+    buildPlatform: $(Agent.OS)
+  condition: always() # Always publish test results
+  displayName: 'Publish test report'

devops/templates/base/run-linter.yml

@@ -0,0 +1,24 @@
+# Template for running linter and other static analysis tools on the code
+parameters:
+- name: pyVersion
+  type: string
+  default: '3.6'
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: ${{ parameters.pyVersion }}
+    addToPath: true
+    architecture: 'x64'
+  displayName: 'Use Python ${{ parameters.pyVersion }}'
+
+- bash: |
+    python -m pip install --upgrade pip setuptools wheel
+    python -m pip install -r requirements-dev.txt
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Install flake8 and other dev dependencies'
+
+- bash: |
+    tox -e flake8
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Run Linter (flake8)'

devops/templates/base/run-tests.yml

@@ -0,0 +1,31 @@
+# Template for running tests on multiple Python versions and platforms
+parameters:
+- name: testType
+  type: string
+  default: all
+  values:
+    - slow
+    - azure
+    - unit
+    - io
+    - all
+
+steps:
+- bash: |
+    if [[ '${{parameters.testType}}' == 'all' ]]
+    then
+      tox -e py 
+    elif [[ '${{parameters.testType}}' == 'unit' ]]
+    then
+      tox -e py -- tests/unit
+    else
+      tox -e py -- -m "${{parameters.testType}}"
+    fi
+  env:
+    # These keys come from azureResourceKeys variable group
+    BLOB_KEY : $(BLOB_KEY) 
+    SEARCH_SERVICE_KEY: $(SEARCH_SERVICE_KEY)
+    COGNITIVE_SERVICE_KEY: $(COGNITIVE_SERVICE_KEY)
+    COMPUTER_VISION_SUBSCRIPTION_KEY: $(COMPUTER_VISION_SUBSCRIPTION_KEY)
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Running (${{parameters.testType}}) Tests'

devops/templates/build_wheel_n_sdist.yml

@@ -0,0 +1,31 @@
+# Template to create wheel and source distribution
+parameters:
+- name: pyVersion
+  default: '3.6'
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: ${{ parameters.pyVersion }}
+    addToPath: true
+    architecture: 'x64'
+  displayName: 'Use Python ${{ parameters.pyVersion }}'
+
+- bash: |
+    python -m pip install --upgrade pip setuptools wheel
+  displayName: 'Update pip and setuptools'
+
+- bash: |
+    python setup.py bdist_wheel
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Build wheel'
+
+- bash: |
+    python setup.py sdist
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Build source distribution'
+
+- bash: |
+    ls dist
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Show artifacts in folder'

devops/templates/install-dependencies.yml

@@ -0,0 +1,7 @@
+# Assume a python version is enabled with "UsePythonVersion@0" task
+steps:
+- bash: |
+    python -m pip install --upgrade pip setuptools wheel
+    python -m pip install -r requirements-dev.txt
+  workingDirectory: $(Build.SourcesDirectory)
+  displayName: 'Install dependencies'

devops/templates/merge-cov-reports.yml

@@ -0,0 +1,43 @@
+# Template to merge several code coverage reports (.coverage*)
+parameters:
+- name: pyVersion
+  default: '3.6'
+
+jobs:
+  - job:
+    displayName: Merge cov reports
+    pool:
+        vmImage: 'ubuntu-latest'
+
+    steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: ${{ parameters.pyVersion }}
+        addToPath: true
+        architecture: 'x64'
+      displayName: 'Use Python ${{ parameters.pyVersion }}'
+
+    - bash: |
+        python -m pip --upgrade pip setuptools
+        python -m pip install coverage
+      workingDirectory: $(Build.SourcesDirectory)
+      displayName: 'Install coverage'
+         
+    # See https://docs.microsoft.com/en-us/azure/devops/pipelines/artifacts/pipeline-artifacts?view=azure-devops&tabs=yaml#multiple-artifacts
+    - download: current
+      patterns: '**/.coverage*'
+
+    - bash: |
+        python -m coverage combine $(Pipeline.Workspace)/**/.coverage*
+        python -m coverage report
+        python -m coverage xml
+      workingDirectory: $(Build.SourcesDirectory)
+      displayName: Show and merge cached coverage report
+
+    - task: PublishCodeCoverageResults@1
+      inputs:
+        codeCoverageTool: Cobertura
+        summaryFileLocation: '$(Build.SourcesDirectory)/coverage.xml'
+      displayName: 'Publish merged code coverage report'
+
+