diff --git a/devops/PyPI-Release.yml b/devops/PyPI-Release.yml new file mode 100644 index 00000000..04781528 --- /dev/null +++ b/devops/PyPI-Release.yml @@ -0,0 +1,200 @@ +# Simplified PyPI release pipeline + +# At queue time, the user selects a Test or Production deployment. The following stages +# then run: +# - Predeployment validation (run a set of tests against the repository) +# - Creates a wheel and stores in Pipeline Artifact +# - Download wheel file from Artifact, pip install, and run tests +# - Upload the wheel to PyPI (Test or Production as specified at queue time) +# - Install from PyPI and run tests + +parameters: +- name: releaseType + displayName: Release Type + type: string + default: Test + values: + - Test + - Production + +variables: + poolImage: "ubuntu-latest" + poolPythonVersion: 3.6 + packageArtifactName: Wheels + versionArtifactName: Version + versionFileName: versionInfo.txt + +trigger: none # No CI build + +pr: none # Not for pull requests + +# ================================================================================================== + +stages: +- stage: PredeploymentValidation + displayName: Predeployment Validation + pool: + vmImage: $(poolImage) + + jobs: + - template: templates/all-tests-job-template.yml + parameters: + platforms: { Linux: ubuntu-latest, MacOS: macos-latest, Windows: windows-latest } + pyVersions: [3.6, 3.7] + installationType: PipLocal + envArtifactStem: PredeployFreeze + envFileStem: redeploy-requirements + +# ================================================================================================== + +- stage: CreateWheel + displayName: Create Wheel Artifact + dependsOn: PredeploymentValidation + pool: + vmImage: $(poolImage) + + variables: + wheelEnvName: WheelEnvironment + + jobs: + - job: CreateWheel + displayName: Build and publish wheel + pool: + vmImage: $(poolImage) + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(poolPythonVersion)' + inputs: + versionSpec: $(poolPythonVersion) + addToPath: true + + - template: templates/create-env-step-template.yml + parameters: + pythonVersion: $(poolPythonVersion) + envInfoArtifact: CreateWheelFreeze + envInfoFileBase: createwheel-freeze + condaEnv: $(wheelEnvName) + + - bash: | + source activate $(wheelEnvName) + pip install --upgrade wheel + displayName: 'Install wheel' + + - bash: | + source activate $(wheelEnvName) + python ./tools/build_wheels.py --version-filename $(versionFilename) + displayName: 'Build wheels' + + - task: PublishPipelineArtifact@1 + displayName: "Publish wheels" + inputs: + path: $(System.DefaultWorkingDirectory)/python/dist + artifact: $(packageArtifactName) + + - task: PublishPipelineArtifact@1 + displayName: "Publish version information file" + inputs: + path: '$(System.DefaultWorkingDirectory)/$(versionFilename)' + artifact: $(versionArtifactName) + +# ================================================================================================== + +- stage: TestWheel + displayName: Test Wheel from Artifact + dependsOn: CreateWheel + pool: + vmImage: $(poolImage) + + jobs: + - template: templates/all-tests-job-template.yml + parameters: + platforms: { Linux: ubuntu-latest, MacOS: macos-latest, Windows: windows-latest } + pyVersions: [3.6, 3.7] + installationType: 'WheelArtifact' + envArtifactStem: TestWheelFreeze + envFileStem: requirements-wheel-test + wheelArtifactName: $(packageArtifactName) + +# ================================================================================================== + +- stage: UploadWheel + displayName: Upload Wheel to PyPI (${{parameters.releaseType}}) + dependsOn: TestWheel + pool: + vmImage: $(poolImage) + + variables: + ${{ if eq(parameters.releaseType, 'Test')}}: + twineConnection: PyPI-Test + twineEndpoint: PyPITest + ${{ if eq(parameters.releaseType, 'Production')}}: + twineConnection: PyPI-Prod + twineEndpoint: PyPIProd + + jobs: + - deployment: 'PyPI_${{parameters.releaseType}}_Upload' + displayName: PyPI ${{parameters.releaseType}} Upload + ${{ if eq(parameters.releaseType, 'Test')}}: + environment: 'PyPI-Test Deployment' + ${{ if eq(parameters.releaseType, 'Production')}}: + environment: 'PyPI Deployment' + pool: + vmImage: $(poolImage) + + strategy: + runOnce: + deploy: + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(poolPythonVersion)' + inputs: + versionSpec: $(poolPythonVersion) + addToPath: true + + - script: pip install twine + displayName: 'Install twine' + + - task: TwineAuthenticate@0 + inputs: + externalFeeds: ${{variables.twineConnection}} + + - script: 'twine upload --verbose -r $(twineEndpoint) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/$(packageArtifactName)/*' + displayName: Upload to ${{parameters.releaseType}} PyPI + + # TODO: Add GitHub Release task, so links in PyPI ReadMe will work without manual intervention (Prod only) + + - job: PyPI_Pause + pool: server + dependsOn: 'PyPI_${{parameters.releaseType}}_Upload' + displayName: PyPI Pause + + steps: + - task: Delay@1 + displayName: "Pause to allow PyPI updates to complete" + inputs: + delayForMinutes: "5" + +# # ================================================================================================== + +- stage: TestFromPyPI + displayName: Test package from ${{parameters.releaseType}} PyPI + dependsOn: UploadWheel + pool: + vmImage: $(poolImage) + + variables: + envInfoArtifact: TestPyPIFreeze + envInfoFileBase: requirements-pypi-test + + jobs: + - template: templates/all-tests-job-template.yml + parameters: + platforms: { Linux: ubuntu-latest, MacOS: macos-latest, Windows: windows-latest } + pyVersions: [3.6, 3.7] + envArtifactStem: TestPyPIFreeze + envFileStem: requirements-pypi-test + installationType: 'PyPI' + targetType: ${{parameters.releaseType}} + versionArtifactName: $(versionArtifactName) + versionArtifactFile: $(versionFileName) \ No newline at end of file diff --git a/devops/templates/create-env-template.yml b/devops/templates/create-env-step-template.yml similarity index 66% rename from devops/templates/create-env-template.yml rename to devops/templates/create-env-step-template.yml index 093cd116..81cd7446 100644 --- a/devops/templates/create-env-template.yml +++ b/devops/templates/create-env-step-template.yml @@ -2,6 +2,13 @@ parameters: - name: pythonVersion type: string +- name: envInfoArtifact + type: string +- name: envInfoFileBase + type: string +- name: envInfoDirectory + type: string + default: environmentInfo - name: condaEnv type: string default: interpret_conda_env @@ -48,3 +55,19 @@ steps: conda install --yes -c conda-forge -n ${{parameters.condaEnv}} papermill pip install nteract-scrapbook displayName: List Jupyter Kernel and fix + + - bash: mkdir ${{parameters.envInfoDirectory}} + displayName: Create directory for environment info + + - bash: | + source activate ${{parameters.condaEnv}} + pip freeze --all > ${{parameters.envInfoFileBase}}-pip.txt + conda list > ${{parameters.envInfoFileBase}}-conda.txt + displayName: "Gather environment information" + workingDirectory: '$(System.DefaultWorkingDirectory)/${{parameters.envInfoDirectory}}' + + - task: PublishPipelineArtifact@1 + displayName: "Publish environment info to artifact ${{parameters.envInfoArtifact}}" + inputs: + path: '$(System.DefaultWorkingDirectory)/${{parameters.envInfoDirectory}}' + artifact: ${{parameters.envInfoArtifact}} diff --git a/devops/templates/environment-info-step-template.yml b/devops/templates/environment-info-step-template.yml deleted file mode 100644 index a7f0cbad..00000000 --- a/devops/templates/environment-info-step-template.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Template to save a list of installed packages to a pipeline Artifact -# in order to help with debugging broken dependencies - -parameters: - condaEnv: - envInfoDirectory: environmentInfo - envInfoArtifact: - envInfoFileBase: - - -steps: - - bash: mkdir ${{parameters.envInfoDirectory}} - displayName: Create directory for environment info - - - bash: | - source activate ${{parameters.condaEnv}} - pip freeze --all > ${{parameters.envInfoFileBase}}-pip.txt - conda list > ${{parameters.envInfoFileBase}}-conda.txt - displayName: "Gather environment information" - workingDirectory: '$(System.DefaultWorkingDirectory)/${{parameters.envInfoDirectory}}' - - - task: PublishPipelineArtifact@1 - displayName: "Publish environment info to artifact ${{parameters.envInfoArtifact}}" - inputs: - path: '$(System.DefaultWorkingDirectory)/${{parameters.envInfoDirectory}}' - artifact: ${{parameters.envInfoArtifact}} \ No newline at end of file diff --git a/devops/templates/package-installation-step-template.yml b/devops/templates/package-installation-step-template.yml index fc4b1136..6aaf2411 100644 --- a/devops/templates/package-installation-step-template.yml +++ b/devops/templates/package-installation-step-template.yml @@ -67,7 +67,7 @@ steps: - task: PowerShell@2 displayName: 'Read version Artifact and set pipeline variable from file contents' inputs: - filePath: scripts/set-variable-from-file.ps1 + filePath: tools/set-variable-from-file.ps1 arguments: "-baseDir $(Build.SourcesDirectory) -subDir . -fileName ${{parameters.versionArtifactFile}} -targetVariable ${{parameters.pipVersionVariable}}" pwsh: true @@ -89,7 +89,7 @@ steps: - bash: | source activate ${{parameters.condaEnv}} - pip install interpret-community*.whl + pip install interpret_community*.whl displayName: "Install interpret-community with pip from local file" diff --git a/devops/templates/test-run-step-template.yml b/devops/templates/test-run-step-template.yml index 37c1441c..0049a346 100644 --- a/devops/templates/test-run-step-template.yml +++ b/devops/templates/test-run-step-template.yml @@ -42,10 +42,12 @@ parameters: steps: - template: conda-path-step-template.yml -- template: create-env-template.yml +- template: create-env-step-template.yml parameters: pythonVersion: ${{parameters.pythonVersion}} condaEnv: ${{parameters.condaEnv}} + envInfoArtifact: ${{parameters.envInfoArtifact}} + envInfoFileBase: ${{parameters.envInfoFileBase}} - bash: | source activate ${{parameters.condaEnv}} @@ -64,12 +66,6 @@ steps: pipVersionVariable: variableForPipVersion wheelArtifactName: ${{parameters.wheelArtifactName}} condaEnv: ${{parameters.condaEnv}} - -- template: environment-info-step-template.yml - parameters: - condaEnv: ${{parameters.condaEnv}} - envInfoArtifact: ${{parameters.envInfoArtifact}} - envInfoFileBase: ${{parameters.envInfoFileBase}} - ${{ if eq(parameters.testRunType, 'Unit')}}: - bash: | diff --git a/python/interpret_community/__init__.py b/python/interpret_community/__init__.py index 29788485..b93e8c5f 100644 --- a/python/interpret_community/__init__.py +++ b/python/interpret_community/__init__.py @@ -32,6 +32,7 @@ def close_handler(): logger.removeHandler(handler) atexit.register(close_handler) +__name__ = "interpret_community" _major = '0' _minor = '14' _patch = '0' diff --git a/python/setup.py b/python/setup.py index b56ccc7d..98c6fc04 100644 --- a/python/setup.py +++ b/python/setup.py @@ -66,7 +66,7 @@ README = f.read() setup( - name='interpret-community', + name=interpret_community.__name__, version=interpret_community.__version__, diff --git a/requirements.txt b/requirements.txt index 38e83887..d5cf1af5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,5 +21,6 @@ pytest-cov nbformat papermill nteract-scrapbook -shap -interpret-core +gevent>=1.3.6 +interpret-core[required]>=0.1.20, <=0.1.21 +shap>=0.20.0, <=0.34.0 \ No newline at end of file diff --git a/tools/_utils.py b/tools/_utils.py new file mode 100644 index 00000000..8a97695e --- /dev/null +++ b/tools/_utils.py @@ -0,0 +1,18 @@ +import logging + +_logger = logging.getLogger(__file__) +logging.basicConfig(level=logging.INFO) + + +class _LogWrapper: + def __init__(self, description): + self._description = description + + def __enter__(self): + _logger.info("Starting %s", self._description) + + def __exit__(self, type, value, traceback): # noqa: A002 + # raise exceptions if any occurred + if value is not None: + raise value + _logger.info("Completed %s", self._description) diff --git a/tools/build_wheels.py b/tools/build_wheels.py new file mode 100644 index 00000000..115ceb75 --- /dev/null +++ b/tools/build_wheels.py @@ -0,0 +1,44 @@ +import argparse +import logging +import subprocess +import sys + +from _utils import _LogWrapper + +_logger = logging.getLogger(__file__) +logging.basicConfig(level=logging.INFO) + + +def build_argument_parser(): + desc = "Build wheels for interpret-community" + + parser = argparse.ArgumentParser(description=desc) + parser.add_argument("--version-filename", + help="The file where the version will be stored.", + required=True) + + return parser + + +def main(argv): + parser = build_argument_parser() + args = parser.parse_args(argv) + + with _LogWrapper("installation of interpret-community"): + subprocess.check_call(["pip", "install", "./python"]) + + with _LogWrapper("Check pip"): + subprocess.check_call(["pip", "freeze"]) + + with _LogWrapper("storing interpret-community version in {}".format(args.version_filename)): + import interpret_community + with open(args.version_filename, 'w') as version_file: + version_file.write(interpret_community.__version__) + + with _LogWrapper("creation of packages"): + subprocess.check_call( + ["python", "./setup.py", "sdist", "bdist_wheel"], cwd="./python/") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tools/set-variable-from-file.ps1 b/tools/set-variable-from-file.ps1 new file mode 100644 index 00000000..40b0ee28 --- /dev/null +++ b/tools/set-variable-from-file.ps1 @@ -0,0 +1,27 @@ +# Script to set a pipeline variable from the contents of a file +# This script is only required for Azure DevOps pipelines. +param( + [Parameter(Mandatory)] + [string]$baseDir, + [Parameter(Mandatory)] + [string]$subDir, + [Parameter(Mandatory)] + [string]$fileName, + [Parameter(Mandatory)] + [string]$targetVariable +) + +Write-Host $baseDir +Write-Host $subDir +Write-Host $fileName + +$srcDir = Join-Path -Resolve $baseDir $subDir +$srcFile = Join-Path -Resolve $srcDir $fileName + +Write-Host "Reading from $srcFile" +Write-Host "Setting variable $targetVariable" + +$contents = Get-Content $srcFile + +Write-Host "##vso[task.setvariable variable=$targetVariable]$contents" +Write-Host "Completed" \ No newline at end of file