PyPI Release Pipeline (#283)

Create an ADO pipeline for doing releases to PyPI, in the pattern of other projects in RAI.

Signed-off-by: Richard Edgar <riedgar@microsoft.com>

devops/PyPI-Release.yml

@@ -0,0 +1,200 @@
+# Simplified PyPI release pipeline
+
+# At queue time, the user selects a Test or Production deployment. The following stages
+# then run:
+# - Predeployment validation (run a set of tests against the repository)
+# - Creates a wheel and stores in Pipeline Artifact
+# - Download wheel file from Artifact, pip install, and run tests
+# - Upload the wheel to PyPI (Test or Production as specified at queue time)
+# - Install from PyPI and run tests
+
+parameters:
+- name: releaseType
+  displayName: Release Type
+  type: string
+  default: Test
+  values:
+  - Test
+  - Production
+
+variables:
+  poolImage: "ubuntu-latest"
+  poolPythonVersion: 3.6
+  packageArtifactName: Wheels
+  versionArtifactName: Version
+  versionFileName: versionInfo.txt
+
+trigger: none # No CI build
+
+pr: none # Not for pull requests
+
+# ==================================================================================================
+
+stages:
+- stage: PredeploymentValidation
+  displayName: Predeployment Validation
+  pool:
+    vmImage: $(poolImage)
+
+  jobs:
+  - template: templates/all-tests-job-template.yml
+    parameters:
+      platforms:  { Linux: ubuntu-latest, MacOS: macos-latest, Windows: windows-latest }
+      pyVersions: [3.6, 3.7]
+      installationType: PipLocal
+      envArtifactStem: PredeployFreeze
+      envFileStem: redeploy-requirements
+
+# ==================================================================================================
+
+- stage: CreateWheel
+  displayName: Create Wheel Artifact
+  dependsOn: PredeploymentValidation
+  pool:
+    vmImage: $(poolImage)
+
+  variables:
+    wheelEnvName: WheelEnvironment
+
+  jobs:
+  - job: CreateWheel
+    displayName: Build and publish wheel
+    pool:
+      vmImage: $(poolImage)
+
+    steps:
+    - task: UsePythonVersion@0
+      displayName: 'Use Python $(poolPythonVersion)'
+      inputs:
+        versionSpec: $(poolPythonVersion)
+        addToPath: true
+
+    - template: templates/create-env-step-template.yml
+      parameters:
+        pythonVersion: $(poolPythonVersion)
+        envInfoArtifact: CreateWheelFreeze
+        envInfoFileBase: createwheel-freeze
+        condaEnv: $(wheelEnvName)
+
+    - bash: |
+        source activate $(wheelEnvName)
+        pip install --upgrade wheel 
+      displayName: 'Install wheel'
+
+    - bash: |
+        source activate $(wheelEnvName)
+        python ./tools/build_wheels.py --version-filename $(versionFilename) 
+      displayName: 'Build wheels'
+
+    - task: PublishPipelineArtifact@1
+      displayName: "Publish wheels"
+      inputs:
+        path: $(System.DefaultWorkingDirectory)/python/dist
+        artifact: $(packageArtifactName)
+
+    - task: PublishPipelineArtifact@1
+      displayName: "Publish version information file"
+      inputs:
+        path: '$(System.DefaultWorkingDirectory)/$(versionFilename)'
+        artifact: $(versionArtifactName)
+
+# ==================================================================================================
+
+- stage: TestWheel
+  displayName: Test Wheel from Artifact
+  dependsOn: CreateWheel
+  pool:
+    vmImage: $(poolImage)
+
+  jobs:
+    - template: templates/all-tests-job-template.yml
+      parameters:
+        platforms:  { Linux: ubuntu-latest, MacOS: macos-latest, Windows: windows-latest }
+        pyVersions: [3.6, 3.7]
+        installationType: 'WheelArtifact'
+        envArtifactStem: TestWheelFreeze
+        envFileStem: requirements-wheel-test
+        wheelArtifactName: $(packageArtifactName)
+
+# ==================================================================================================
+
+- stage: UploadWheel
+  displayName: Upload Wheel to PyPI (${{parameters.releaseType}})
+  dependsOn: TestWheel
+  pool:
+    vmImage: $(poolImage)
+
+  variables:
+    ${{ if eq(parameters.releaseType, 'Test')}}:
+      twineConnection: PyPI-Test
+      twineEndpoint: PyPITest
+    ${{ if eq(parameters.releaseType, 'Production')}}:
+      twineConnection: PyPI-Prod
+      twineEndpoint: PyPIProd
+
+  jobs:
+  - deployment: 'PyPI_${{parameters.releaseType}}_Upload'
+    displayName: PyPI ${{parameters.releaseType}} Upload
+    ${{ if eq(parameters.releaseType, 'Test')}}:
+      environment: 'PyPI-Test Deployment'
+    ${{ if eq(parameters.releaseType, 'Production')}}:
+      environment: 'PyPI Deployment'
+    pool:
+      vmImage: $(poolImage)
+
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+          - task: UsePythonVersion@0
+            displayName: 'Use Python $(poolPythonVersion)'
+            inputs:
+              versionSpec: $(poolPythonVersion)
+              addToPath: true
+
+          - script: pip install twine
+            displayName: 'Install twine'
+
+          - task: TwineAuthenticate@0
+            inputs:
+              externalFeeds: ${{variables.twineConnection}}
+
+          - script: 'twine upload --verbose -r $(twineEndpoint) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/$(packageArtifactName)/*'
+            displayName: Upload to ${{parameters.releaseType}} PyPI
+
+        # TODO: Add GitHub Release task, so links in PyPI ReadMe will work without manual intervention (Prod only)
+
+  - job: PyPI_Pause
+    pool: server
+    dependsOn: 'PyPI_${{parameters.releaseType}}_Upload'
+    displayName: PyPI Pause
+
+    steps:
+    - task: Delay@1
+      displayName: "Pause to allow PyPI updates to complete"
+      inputs:
+        delayForMinutes: "5"
+
+# # ==================================================================================================
+
+- stage: TestFromPyPI
+  displayName: Test package from ${{parameters.releaseType}} PyPI
+  dependsOn: UploadWheel
+  pool:
+    vmImage: $(poolImage)
+
+  variables:
+    envInfoArtifact: TestPyPIFreeze
+    envInfoFileBase: requirements-pypi-test
+
+  jobs:
+  - template: templates/all-tests-job-template.yml
+    parameters:
+      platforms:  { Linux: ubuntu-latest, MacOS: macos-latest, Windows: windows-latest }
+      pyVersions: [3.6, 3.7]
+      envArtifactStem: TestPyPIFreeze
+      envFileStem: requirements-pypi-test
+      installationType: 'PyPI'
+      targetType: ${{parameters.releaseType}}
+      versionArtifactName: $(versionArtifactName)
+      versionArtifactFile: $(versionFileName)

devops/templates/create-env-template.yml → devops/templates/create-env-step-template.yml

@@ -2,6 +2,13 @@
 parameters:
 - name: pythonVersion
   type: string
+- name: envInfoArtifact
+  type: string
+- name: envInfoFileBase
+  type: string
+- name: envInfoDirectory
+  type: string
+  default: environmentInfo
 - name: condaEnv
   type: string
   default: interpret_conda_env
@@ -48,3 +55,19 @@ steps:
       conda install --yes -c conda-forge -n ${{parameters.condaEnv}} papermill 
       pip install nteract-scrapbook
     displayName: List Jupyter Kernel and fix
+
+  - bash: mkdir ${{parameters.envInfoDirectory}}
+    displayName: Create directory for environment info
+
+  - bash: |
+        source activate ${{parameters.condaEnv}}
+        pip freeze --all > ${{parameters.envInfoFileBase}}-pip.txt
+        conda list > ${{parameters.envInfoFileBase}}-conda.txt
+    displayName: "Gather environment information"
+    workingDirectory: '$(System.DefaultWorkingDirectory)/${{parameters.envInfoDirectory}}'
+
+  - task: PublishPipelineArtifact@1
+    displayName: "Publish environment info to artifact ${{parameters.envInfoArtifact}}"
+    inputs:
+      path: '$(System.DefaultWorkingDirectory)/${{parameters.envInfoDirectory}}'
+      artifact: ${{parameters.envInfoArtifact}}

devops/templates/package-installation-step-template.yml

@@ -67,7 +67,7 @@ steps:
     - task: PowerShell@2
       displayName: 'Read version Artifact and set pipeline variable from file contents'
       inputs:
-        filePath: scripts/set-variable-from-file.ps1
+        filePath: tools/set-variable-from-file.ps1
         arguments: "-baseDir $(Build.SourcesDirectory) -subDir . -fileName ${{parameters.versionArtifactFile}} -targetVariable ${{parameters.pipVersionVariable}}"
         pwsh: true
 
@@ -89,7 +89,7 @@ steps:
 
     - bash: |
         source activate ${{parameters.condaEnv}}
-        pip install interpret-community*.whl
+        pip install interpret_community*.whl
       displayName: "Install interpret-community with pip from local file"
 
       

devops/templates/test-run-step-template.yml

@@ -42,10 +42,12 @@ parameters:
 steps:
 - template: conda-path-step-template.yml
 
-- template: create-env-template.yml
+- template: create-env-step-template.yml
   parameters:
     pythonVersion: ${{parameters.pythonVersion}}
     condaEnv: ${{parameters.condaEnv}}
+    envInfoArtifact: ${{parameters.envInfoArtifact}}
+    envInfoFileBase: ${{parameters.envInfoFileBase}}
 
 - bash: |
       source activate ${{parameters.condaEnv}}
@@ -64,12 +66,6 @@ steps:
     pipVersionVariable: variableForPipVersion
     wheelArtifactName: ${{parameters.wheelArtifactName}}
     condaEnv: ${{parameters.condaEnv}}
-
-- template: environment-info-step-template.yml
-  parameters:
-    condaEnv: ${{parameters.condaEnv}}
-    envInfoArtifact: ${{parameters.envInfoArtifact}}
-    envInfoFileBase: ${{parameters.envInfoFileBase}}
 
 - ${{ if eq(parameters.testRunType, 'Unit')}}:
   - bash: |

python/interpret_community/__init__.py

@@ -32,6 +32,7 @@ def close_handler():
         logger.removeHandler(handler)
     atexit.register(close_handler)
 
+__name__ = "interpret_community"
 _major = '0'
 _minor = '14'
 _patch = '0'

python/setup.py

@@ -66,7 +66,7 @@
     README = f.read()
 
 setup(
-    name='interpret-community',
+    name=interpret_community.__name__,
 
     version=interpret_community.__version__,
 

requirements.txt

@@ -21,5 +21,6 @@ pytest-cov
 nbformat
 papermill
 nteract-scrapbook
-shap
-interpret-core
+gevent>=1.3.6
+interpret-core[required]>=0.1.20, <=0.1.21
+shap>=0.20.0, <=0.34.0

tools/_utils.py

@@ -0,0 +1,18 @@
+import logging
+
+_logger = logging.getLogger(__file__)
+logging.basicConfig(level=logging.INFO)
+
+
+class _LogWrapper:
+    def __init__(self, description):
+        self._description = description
+
+    def __enter__(self):
+        _logger.info("Starting %s", self._description)
+
+    def __exit__(self, type, value, traceback):  # noqa: A002
+        # raise exceptions if any occurred
+        if value is not None:
+            raise value
+        _logger.info("Completed %s", self._description)

tools/build_wheels.py

@@ -0,0 +1,44 @@
+import argparse
+import logging
+import subprocess
+import sys
+
+from _utils import _LogWrapper
+
+_logger = logging.getLogger(__file__)
+logging.basicConfig(level=logging.INFO)
+
+
+def build_argument_parser():
+    desc = "Build wheels for interpret-community"
+
+    parser = argparse.ArgumentParser(description=desc)
+    parser.add_argument("--version-filename",
+                        help="The file where the version will be stored.",
+                        required=True)
+
+    return parser
+
+
+def main(argv):
+    parser = build_argument_parser()
+    args = parser.parse_args(argv)
+
+    with _LogWrapper("installation of interpret-community"):
+        subprocess.check_call(["pip", "install", "./python"])
+
+    with _LogWrapper("Check pip"):
+        subprocess.check_call(["pip", "freeze"])
+
+    with _LogWrapper("storing interpret-community version in {}".format(args.version_filename)):
+        import interpret_community
+        with open(args.version_filename, 'w') as version_file:
+            version_file.write(interpret_community.__version__)
+
+    with _LogWrapper("creation of packages"):
+        subprocess.check_call(
+            ["python", "./setup.py", "sdist", "bdist_wheel"], cwd="./python/")
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])