diff --git a/README.rst b/README.rst index 41fe762c0..269dd2afa 100644 --- a/README.rst +++ b/README.rst @@ -37,13 +37,13 @@ Distribution includes Intel® SDC for Python 3.6 and Python 3.7 for Windows and Intel® SDC conda package can be installed using the steps below:: - > conda create -n sdc-env python=<3.7 or 3.6> pyarrow=2.0.0 pandas=1.2.0 -c anaconda -c conda-forge + > conda create -n sdc-env python=<3.7 or 3.6> -c anaconda -c conda-forge > conda activate sdc-env > conda install sdc -c intel/label/beta -c intel -c defaults -c conda-forge --override-channels Intel® SDC wheel package can be installed using the steps below:: - > conda create -n sdc-env python=<3.7 or 3.6> pip pyarrow=2.0.0 pandas=1.2.0 -c anaconda -c conda-forge + > conda create -n sdc-env python=<3.7 or 3.6> pip -c anaconda -c conda-forge > conda activate sdc-env > pip install --index-url https://pypi.anaconda.org/intel/label/beta/simple --extra-index-url https://pypi.anaconda.org/intel/simple --extra-index-url https://pypi.org/simple sdc diff --git a/buildscripts/azure/template-linux-macos.yml b/buildscripts/azure/template-linux-macos.yml index f4cc8f712..277cc69a6 100644 --- a/buildscripts/azure/template-linux-macos.yml +++ b/buildscripts/azure/template-linux-macos.yml @@ -43,3 +43,9 @@ jobs: python buildscripts/build_doc.py --python=$PYTHON_VER --sdc-channel=./sdc-build displayName: 'Build Intel SDC documentation' continueOnError: ${{ parameters.allowFailure }} + + - script: | + source $HOME/miniconda3/bin/activate + python buildscripts/test_conda_install.py --python=$PYTHON_VER --channels="-c anaconda -c conda-forge" + displayName: 'Test SDC installation from conda' + continueOnError: ${{ parameters.allowFailure }} diff --git a/buildscripts/azure/template-windows.yml b/buildscripts/azure/template-windows.yml index 2991b25c2..45e469262 100644 --- a/buildscripts/azure/template-windows.yml +++ b/buildscripts/azure/template-windows.yml @@ -36,3 +36,9 @@ jobs: python buildscripts\\run_examples.py --python=%PYTHON_VER% --sdc-channel=.\\sdc-build displayName: 'Run Intel SDC examples' continueOnError: ${{ parameters.allowFailure }} + + - script: | + call C:\\Miniconda\\Scripts\activate.bat + python buildscripts\\test_conda_install.py --python=%PYTHON_VER% --channels="-c anaconda -c conda-forge" + displayName: 'Test SDC installation from conda' + continueOnError: ${{ parameters.allowFailure }} diff --git a/buildscripts/build_doc.py b/buildscripts/build_doc.py index d2477daaf..74fad7a8e 100644 --- a/buildscripts/build_doc.py +++ b/buildscripts/build_doc.py @@ -88,12 +88,13 @@ def publish_doc(sdc_utils): parser = argparse.ArgumentParser() parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'], help='Python version, default = 3.7') + parser.add_argument('--channels', default=None, help='Default env channels') parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel') parser.add_argument('--publish', action='store_true', help='Publish documentation to sdc-doc') args = parser.parse_args() - sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel) + sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel) sdc_utils.log_info('Build Intel(R) SDC documentation', separate=True) sdc_utils.log_info(sdc_utils.line_double) sdc_utils.create_environment(['sphinx', 'sphinxcontrib-programoutput']) diff --git a/buildscripts/run_benchmarks.py b/buildscripts/run_benchmarks.py index 1000784eb..b9c35ac5b 100644 --- a/buildscripts/run_benchmarks.py +++ b/buildscripts/run_benchmarks.py @@ -47,6 +47,7 @@ def run_benchmarks(sdc_utils, args_list, num_threads_list): parser = argparse.ArgumentParser() parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'], help='Python version, default = 3.7') + parser.add_argument('--channels', default=None, help='Default env channels') parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel') parser.add_argument('--args-list', required=True, nargs='+', help='List of arguments sets for benchmarks') parser.add_argument('--num-threads-list', required=True, nargs='+', @@ -54,7 +55,7 @@ def run_benchmarks(sdc_utils, args_list, num_threads_list): args = parser.parse_args() - sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel) + sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel) sdc_utils.log_info('Run Intel(R) SDC benchmarks', separate=True) sdc_utils.log_info(sdc_utils.line_double) sdc_utils.create_environment(['openpyxl', 'xlrd']) diff --git a/buildscripts/run_examples.py b/buildscripts/run_examples.py index 633c70edd..32484d4e1 100644 --- a/buildscripts/run_examples.py +++ b/buildscripts/run_examples.py @@ -119,11 +119,12 @@ def run_examples(sdc_utils): parser = argparse.ArgumentParser() parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'], help='Python version, default = 3.7') + parser.add_argument('--channels', default=None, help='Default env channels') parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel') args = parser.parse_args() - sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel) + sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel) sdc_utils.log_info('Run Intel(R) SDC examples', separate=True) sdc_utils.log_info(sdc_utils.line_double) sdc_utils.create_environment() diff --git a/buildscripts/test.py b/buildscripts/test.py index d16739a61..e0a1a4f72 100644 --- a/buildscripts/test.py +++ b/buildscripts/test.py @@ -48,11 +48,12 @@ def run_tests(sdc_utils): parser = argparse.ArgumentParser() parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'], help='Python version, default = 3.7') - parser.add_argument('--sdc-channel', required=True, help='Local Intel SDC channel') + parser.add_argument('--channels', default=None, help='Default env channels') + parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel') args = parser.parse_args() - sdc_utils = SDC_Build_Utilities(args.python, args.sdc_channel) + sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel) sdc_utils.log_info('Run Intel(R) SDC tests', separate=True) sdc_utils.log_info(sdc_utils.line_double) sdc_utils.create_environment(['conda-build']) diff --git a/buildscripts/test_conda_install.py b/buildscripts/test_conda_install.py new file mode 100644 index 000000000..c16a75f53 --- /dev/null +++ b/buildscripts/test_conda_install.py @@ -0,0 +1,63 @@ +# ***************************************************************************** +# Copyright (c) 2019-2021, Intel Corporation All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + + +import argparse +import os +import shutil +import traceback +import re + +from pathlib import Path +from utilities import SDC_Build_Utilities + + +def check_sdc_installed(sdc_utils, sdc_package): + cmd_output = sdc_utils.get_command_output('conda list sdc') + pattern = sdc_package.replace('=', r'\s+') + return re.search(pattern, cmd_output) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--python', default='3.7', choices=['3.6', '3.7', '3.8'], + help='Python version, default = 3.7') + parser.add_argument('--channels', default=None, help='Default env channels') + parser.add_argument('--sdc-channel', default=None, help='Intel SDC channel') + + args = parser.parse_args() + + sdc_utils = SDC_Build_Utilities(args.python, args.channels, args.sdc_channel) + sdc_utils.log_info('Test Intel(R) SDC conda install', separate=True) + sdc_utils.log_info(sdc_utils.line_double) + sdc_utils.create_environment() + sdc_package = f'sdc={sdc_utils.get_sdc_version_from_channel()}' + + # channels list is aligned with install instruction in README.rst + install_channels = "-c intel/label/beta -c intel -c defaults -c conda-forge" + sdc_utils.install_conda_package([sdc_package], channels=install_channels) + + assert check_sdc_installed(sdc_utils, sdc_package), "SDC package was not installed" diff --git a/buildscripts/utilities.py b/buildscripts/utilities.py index 440c64ec0..cf770b6dd 100644 --- a/buildscripts/utilities.py +++ b/buildscripts/utilities.py @@ -25,7 +25,6 @@ # ***************************************************************************** - import json import os import platform @@ -41,7 +40,7 @@ class SDC_Build_Utilities: - def __init__(self, python, sdc_local_channel=None): + def __init__(self, python, channels=None, sdc_channel=None): self.src_path = Path(__file__).resolve().parent.parent self.env_name = 'sdc_env' self.python = python @@ -52,10 +51,14 @@ def __init__(self, python, sdc_local_channel=None): self.line_single = '-'*80 # Set channels - self.channel_list = ['-c', 'defaults', '-c', 'conda-forge'] - if sdc_local_channel: - sdc_local_channel = Path(sdc_local_channel).resolve().as_uri() - self.channel_list = ['-c', sdc_local_channel] + self.channel_list + build_channels = ['-c', 'defaults', '-c', 'conda-forge'] + self.channel_list = build_channels if channels is None else channels.split() + if sdc_channel: + self.sdc_channel = Path(sdc_channel).resolve().as_uri() + self.channel_list = ['-c', self.sdc_channel] + self.channel_list + else: + self.sdc_channel = 'intel/label/beta' + # keep SDC channel but do not add it to env channels self.channels = ' '.join(self.channel_list) # Conda activate command and conda croot (build) folder @@ -92,12 +95,13 @@ def create_environment(self, packages_list=[]): return - def install_conda_package(self, packages_list): + def install_conda_package(self, packages_list, channels=None): assert type(packages_list) == list, 'Argument should be a list' self.log_info(f'Install {" ".join(packages_list)} to {self.env_name} conda environment') install_args = ['-n', self.env_name] - install_args += self.channel_list + ['--override-channels', '-q', '-y'] + packages_list + replace_channels = channels.split() if channels else self.channel_list + install_args += replace_channels + ['--override-channels', '-q', '-y'] + packages_list self.log_info(self.__run_conda_command(Conda_Commands.INSTALL, install_args)) return @@ -140,12 +144,11 @@ def log_info(self, msg, separate=False): def get_sdc_version_from_channel(self): python_version = 'py' + self.python.replace('.', '') - # Get Intel SDC version from first channel in channel_list - search_args = ['sdc', '-c', self.channel_list[1], '--override-channels', '--json'] + search_args = ['sdc', '-c', self.sdc_channel, '--override-channels', '--json'] search_result = self.__run_conda_command(Conda_Commands.SEARCH, search_args) repo_data = json.loads(search_result) - for package_data in repo_data['sdc']: + for package_data in reversed(repo_data['sdc']): sdc_version = package_data['version'] sdc_build = package_data['build'] if python_version in sdc_build: