From 1900127a5f88cb36c2f0fc3e140069251170c642 Mon Sep 17 00:00:00 2001 From: Nelle Varoquaux Date: Fri, 4 Mar 2016 18:33:36 +0100 Subject: [PATCH 1/4] DOC - added gallery --- doc/Makefile | 4 ++++ doc/install.rst | 2 ++ doc/modules/normalization.rst | 5 +++++ doc/themes/iced/layout.html | 5 +++-- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index 3cd42d1..5ccc255 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -179,4 +179,8 @@ pseudoxml: @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." deploy: + rsync -avz _build/html/ nvaroquaux@cbio.ensmp.fr:public_html/iced-dev/ + +deploy-release: rsync -avz _build/html/ nvaroquaux@cbio.ensmp.fr:public_html/iced/ + diff --git a/doc/install.rst b/doc/install.rst index 94247dc..fbd141e 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -13,6 +13,8 @@ The dependencies are: - scipy (>= 0.7) - argparse if python <2.7 +In addition, pandas is recommanded for fast data loading. + All of these dependencies can be installed at once using `Anaconda `_ diff --git a/doc/modules/normalization.rst b/doc/modules/normalization.rst index b696e20..5326d05 100644 --- a/doc/modules/normalization.rst +++ b/doc/modules/normalization.rst @@ -8,3 +8,8 @@ The :mod:`normalization` submodule contains normalization methods based on a matrix scaling approach. .. currentmodule:: iced.normalization + +.. images:: ../auto_examples/normalization/images/plot_ice_normalization.py + :target: ../auto_examples/normalization/plot_ice_normalization.html + :align: right + :scale: 100% diff --git a/doc/themes/iced/layout.html b/doc/themes/iced/layout.html index f16fb48..ff22f72 100644 --- a/doc/themes/iced/layout.html +++ b/doc/themes/iced/layout.html @@ -44,8 +44,9 @@
  • Home
  • Installation
  • Documentation
    • - {#
  • Data
    • -
  • Results
    • # #} +
  • Gallery
    • +
  • Tutorials
    • + {#
  • Results
    • # #} {%- endblock -%} From 2327e1639d5762aa6423689b6b5c6d9bcef51041 Mon Sep 17 00:00:00 2001 From: Nelle Varoquaux Date: Tue, 16 Feb 2016 11:28:58 +0100 Subject: [PATCH 2/4] DOC: improved documentation --- doc/Makefile | 2 +- doc/modules/classes.rst | 24 ++++++++++++++++++++++++ doc/modules/datasets.rst | 10 ++++++++++ doc/sphinxext/gen_rst.py | 5 ++++- doc/themes/iced/layout.html | 2 +- doc/tutorial/basic/tutorial.rst | 4 ++++ doc/tutorial/index.rst | 20 ++++++++++++++++++++ examples/normalization/README.txt | 6 ++++++ iced/__init__.py | 3 ++- setup.py | 2 +- 10 files changed, 73 insertions(+), 5 deletions(-) create mode 100644 doc/modules/datasets.rst create mode 100644 doc/tutorial/basic/tutorial.rst create mode 100644 doc/tutorial/index.rst create mode 100644 examples/normalization/README.txt diff --git a/doc/Makefile b/doc/Makefile index 5ccc255..f3dc8dc 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -179,7 +179,7 @@ pseudoxml: @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." deploy: - rsync -avz _build/html/ nvaroquaux@cbio.ensmp.fr:public_html/iced-dev/ + rsync -avz _build/html/ nvaroquaux@cbio.ensmp.fr:public_html/iced/dev/ deploy-release: rsync -avz _build/html/ nvaroquaux@cbio.ensmp.fr:public_html/iced/ diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst index 08f2ef0..0607d3f 100644 --- a/doc/modules/classes.rst +++ b/doc/modules/classes.rst @@ -48,3 +48,27 @@ Functions filter.filter_high_counts .. _filter_ref: + + +:mod:`iced.datasets`: Datasets +=============================================== + +.. automodule:: iced.datasets + :no-members: + :no-inherited-members: + + +Functions +--------- +.. currentmodule:: iced + +.. autosummary:: + :toctree: generated/ + :template: function.rst + + datasets.load_sample_yeast + + +.. _datasets_ref: + + diff --git a/doc/modules/datasets.rst b/doc/modules/datasets.rst new file mode 100644 index 0000000..bc19b66 --- /dev/null +++ b/doc/modules/datasets.rst @@ -0,0 +1,10 @@ +.. _datasets: + +============== +Datasets +============== + +The :mod:`datasets` submodule contains utilities to download and load datasets +in numpy format. + +.. currentmodule:: iced.datasets diff --git a/doc/sphinxext/gen_rst.py b/doc/sphinxext/gen_rst.py index 2a0a371..339e21d 100644 --- a/doc/sphinxext/gen_rst.py +++ b/doc/sphinxext/gen_rst.py @@ -502,7 +502,7 @@ def sort_key(a): """) # clear at the end of the section # modules for which we embed links into example code -DOCMODULES = ['sklearn', 'matplotlib', 'numpy', 'scipy', "pastis"] +DOCMODULES = ['iced', 'sklearn', 'matplotlib', 'numpy', 'scipy', "pastis"] def make_thumbnail(in_fname, out_fname, width, height): @@ -774,6 +774,9 @@ def embed_code_links(app, exception): doc_resolvers['scipy'] = SphinxDocLinkResolver( 'http://docs.scipy.org/doc/scipy-0.11.0/reference') + doc_resolvers['iced'] = SphinxDocLinkResolver( + 'http://cbio.ensmp.fr/iced') + example_dir = os.path.join(app.builder.srcdir, 'auto_examples') html_example_dir = os.path.abspath(os.path.join(app.builder.outdir, 'auto_examples')) diff --git a/doc/themes/iced/layout.html b/doc/themes/iced/layout.html index ff22f72..17b5818 100644 --- a/doc/themes/iced/layout.html +++ b/doc/themes/iced/layout.html @@ -44,8 +44,8 @@
  • Home
  • Installation
  • Documentation
    • -
  • Gallery
  • Tutorials
    • +
  • Gallery
    • {#
  • Results
    • # #} diff --git a/doc/tutorial/basic/tutorial.rst b/doc/tutorial/basic/tutorial.rst new file mode 100644 index 0000000..a18d537 --- /dev/null +++ b/doc/tutorial/basic/tutorial.rst @@ -0,0 +1,4 @@ +.. _introduction: + +An introduction to contact counts normalization with iced +========================================================= diff --git a/doc/tutorial/index.rst b/doc/tutorial/index.rst new file mode 100644 index 0000000..321b3cd --- /dev/null +++ b/doc/tutorial/index.rst @@ -0,0 +1,20 @@ +.. _tutorial_menu: + + +================ +Iced tutorial +================ + + + +.. note:: **Doctest Mode** + + The code-examples in the above tutorials are written in a + *python-console* format. If you wish to easily execute these examples + in **IPython**, use:: + + %doctest_mode + + in the IPython-console. You can then simply copy and paste the examples + directly into IPython without having to worry about removing the **>>>** + manually. diff --git a/examples/normalization/README.txt b/examples/normalization/README.txt new file mode 100644 index 0000000..8bd3641 --- /dev/null +++ b/examples/normalization/README.txt @@ -0,0 +1,6 @@ +.. _normalization_examples: + +Normalization +------------- + +Examples concerning the :mod:`iced.normalization` module. diff --git a/iced/__init__.py b/iced/__init__.py index 34f9a3c..f43a992 100644 --- a/iced/__init__.py +++ b/iced/__init__.py @@ -1,5 +1,6 @@ from . import normalization from . import filter from . import io +from . import datasets -__version__ = "0.1-git" +__version__ = "0.2.2-git" diff --git a/setup.py b/setup.py index 8ac638c..d320821 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ DESCRIPTION = 'ICE normalization' MAINTAINER = 'Nelle Varoquaux' MAINTAINER_EMAIL = 'nelle.varoquaux@gmail.com' -VERSION = '0.2.1' +VERSION = '0.2.2-git' def configuration(parent_package='', top_path=None): From 9b7f9220c90710d11d71abb41cec0d7d6c2998ed Mon Sep 17 00:00:00 2001 From: Nelle Varoquaux Date: Fri, 4 Mar 2016 23:49:36 +0100 Subject: [PATCH 3/4] FIX remove the installation of io as independent module --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index d320821..b2c5592 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,6 @@ def configuration(parent_package='', top_path=None): config = Configuration(None, parent_package, top_path) config.add_subpackage('iced') - config.add_subpackage('iced/io') return config From b9fc65c5ea297118eea2f01dad83810de37fa9e6 Mon Sep 17 00:00:00 2001 From: Nelle Varoquaux Date: Fri, 4 Mar 2016 23:55:57 +0100 Subject: [PATCH 4/4] ENH add appveyor --- appveyor.yml | 93 ++++++ build_tools/appveyor/install.ps1 | 229 +++++++++++++++ build_tools/appveyor/requirements.txt | 15 + build_tools/appveyor/run_with_env.cmd | 88 ++++++ build_tools/circle/build_doc.sh | 55 ++++ build_tools/circle/check_build_doc.py | 66 +++++ build_tools/circle/push_doc.sh | 33 +++ build_tools/cythonize.py | 198 +++++++++++++ build_tools/travis/after_success.sh | 19 ++ build_tools/travis/install.sh | 112 ++++++++ build_tools/travis/test_script.sh | 39 +++ .../windows/windows_testing_downloader.ps1 | 270 ++++++++++++++++++ 12 files changed, 1217 insertions(+) create mode 100644 appveyor.yml create mode 100644 build_tools/appveyor/install.ps1 create mode 100644 build_tools/appveyor/requirements.txt create mode 100644 build_tools/appveyor/run_with_env.cmd create mode 100755 build_tools/circle/build_doc.sh create mode 100644 build_tools/circle/check_build_doc.py create mode 100755 build_tools/circle/push_doc.sh create mode 100755 build_tools/cythonize.py create mode 100755 build_tools/travis/after_success.sh create mode 100755 build_tools/travis/install.sh create mode 100755 build_tools/travis/test_script.sh create mode 100644 build_tools/windows/windows_testing_downloader.ps1 diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..3ceb97b --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,93 @@ +# AppVeyor.com is a Continuous Integration service to build and run tests under +# Windows +# https://ci.appveyor.com/project/iced-ci/iced + +environment: + global: + # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the + # /E:ON and /V:ON options are not enabled in the batch script interpreter + # See: http://stackoverflow.com/a/13751649/163740 + CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\build_tools\\appveyor\\run_with_env.cmd" + WHEELHOUSE_UPLOADER_USERNAME: iced-appveyor + WHEELHOUSE_UPLOADER_SECRET: + secure: BQm8KfEj6v2Y+dQxb2syQvTFxDnHXvaNktkLcYSq7jfbTOO6eH9n09tfQzFUVcWZ + + # Make sure we don't download large datasets when running the test on + # continuous integration platform + iced_SKIP_NETWORK_TESTS: 1 + + matrix: + - PYTHON: "C:\\Python27" + PYTHON_VERSION: "2.7.8" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python27-x64" + PYTHON_VERSION: "2.7.8" + PYTHON_ARCH: "64" + + - PYTHON: "C:\\Python35" + PYTHON_VERSION: "3.5.0" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5.0" + PYTHON_ARCH: "64" + + + +install: + # Install Python (from the official .msi of http://python.org) and pip when + # not already installed. + - "powershell ./build_tools/appveyor/install.ps1" + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" + - "python -m pip install -U pip" + + # Check that we have the expected version and architecture for Python + - "python --version" + - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" + - "pip --version" + + # Install the build and runtime dependencies of the project. + - "%CMD_IN_ENV% pip install --timeout=60 --trusted-host 28daf2247a33ed269873-7b1aad3fab3cc330e1fd9d109892382a.r6.cf2.rackcdn.com -r build_tools/appveyor/requirements.txt" + - "%CMD_IN_ENV% python setup.py bdist_wheel bdist_wininst -b doc/logos/scikit-learn-logo.bmp" + - ps: "ls dist" + + # Install the generated wheel package to test it + - "pip install --pre --no-index --find-links dist/ scikit-learn" + +# Not a .NET project, we build scikit-learn in the install step instead +build: false + +test_script: + # Change to a non-source folder to make sure we run the tests on the + # installed library. + - "mkdir empty_folder" + - "cd empty_folder" + + - "python -c \"import nose; nose.main()\" --with-timer --timer-top-n 20 -s -v iced" + + # Move back to the project folder + - "cd .." + +artifacts: + # Archive the generated wheel package in the ci.appveyor.com build report. + - path: dist\* + +on_success: + # Upload the generated wheel package to Rackspace + # On Windows, Apache Libcloud cannot find a standard CA cert bundle so we + # disable the ssl checks. + - "python -m wheelhouse_uploader upload --no-ssl-check --local-folder=dist iced-windows-wheels" + +notifications: + - provider: Webhook + url: https://webhooks.gitter.im/e/0dc8e57cd38105aeb1b4 + on_build_success: false + on_build_failure: True + +cache: + # Use the appveyor cache to avoid re-downloading large archives such + # the MKL numpy and scipy wheels mirrored on a rackspace cloud + # container, speed up the appveyor jobs and reduce bandwidth + # usage on our rackspace account. + - '%APPDATA%\pip\Cache' diff --git a/build_tools/appveyor/install.ps1 b/build_tools/appveyor/install.ps1 new file mode 100644 index 0000000..160ba55 --- /dev/null +++ b/build_tools/appveyor/install.ps1 @@ -0,0 +1,229 @@ +# Sample script to install Python and pip under Windows +# Authors: Olivier Grisel, Jonathan Helmus, Kyle Kastner, and Alex Willmer +# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ + +$MINICONDA_URL = "http://repo.continuum.io/miniconda/" +$BASE_URL = "https://www.python.org/ftp/python/" +$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" +$GET_PIP_PATH = "C:\get-pip.py" + +$PYTHON_PRERELEASE_REGEX = @" +(?x) +(?\d+) +\. +(?\d+) +\. +(?\d+) +(?[a-z]{1,2}\d+) +"@ + + +function Download ($filename, $url) { + $webclient = New-Object System.Net.WebClient + + $basedir = $pwd.Path + "\" + $filepath = $basedir + $filename + if (Test-Path $filename) { + Write-Host "Reusing" $filepath + return $filepath + } + + # Download and retry up to 3 times in case of network transient errors. + Write-Host "Downloading" $filename "from" $url + $retry_attempts = 2 + for ($i = 0; $i -lt $retry_attempts; $i++) { + try { + $webclient.DownloadFile($url, $filepath) + break + } + Catch [Exception]{ + Start-Sleep 1 + } + } + if (Test-Path $filepath) { + Write-Host "File saved at" $filepath + } else { + # Retry once to get the error message if any at the last try + $webclient.DownloadFile($url, $filepath) + } + return $filepath +} + + +function ParsePythonVersion ($python_version) { + if ($python_version -match $PYTHON_PRERELEASE_REGEX) { + return ([int]$matches.major, [int]$matches.minor, [int]$matches.micro, + $matches.prerelease) + } + $version_obj = [version]$python_version + return ($version_obj.major, $version_obj.minor, $version_obj.build, "") +} + + +function DownloadPython ($python_version, $platform_suffix) { + $major, $minor, $micro, $prerelease = ParsePythonVersion $python_version + + if (($major -le 2 -and $micro -eq 0) ` + -or ($major -eq 3 -and $minor -le 2 -and $micro -eq 0) ` + ) { + $dir = "$major.$minor" + $python_version = "$major.$minor$prerelease" + } else { + $dir = "$major.$minor.$micro" + } + + if ($prerelease) { + if (($major -le 2) ` + -or ($major -eq 3 -and $minor -eq 1) ` + -or ($major -eq 3 -and $minor -eq 2) ` + -or ($major -eq 3 -and $minor -eq 3) ` + ) { + $dir = "$dir/prev" + } + } + + if (($major -le 2) -or ($major -le 3 -and $minor -le 4)) { + $ext = "msi" + if ($platform_suffix) { + $platform_suffix = ".$platform_suffix" + } + } else { + $ext = "exe" + if ($platform_suffix) { + $platform_suffix = "-$platform_suffix" + } + } + + $filename = "python-$python_version$platform_suffix.$ext" + $url = "$BASE_URL$dir/$filename" + $filepath = Download $filename $url + return $filepath +} + + +function InstallPython ($python_version, $architecture, $python_home) { + Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home + if (Test-Path $python_home) { + Write-Host $python_home "already exists, skipping." + return $false + } + if ($architecture -eq "32") { + $platform_suffix = "" + } else { + $platform_suffix = "amd64" + } + $installer_path = DownloadPython $python_version $platform_suffix + $installer_ext = [System.IO.Path]::GetExtension($installer_path) + Write-Host "Installing $installer_path to $python_home" + $install_log = $python_home + ".log" + if ($installer_ext -eq '.msi') { + InstallPythonMSI $installer_path $python_home $install_log + } else { + InstallPythonEXE $installer_path $python_home $install_log + } + if (Test-Path $python_home) { + Write-Host "Python $python_version ($architecture) installation complete" + } else { + Write-Host "Failed to install Python in $python_home" + Get-Content -Path $install_log + Exit 1 + } +} + + +function InstallPythonEXE ($exepath, $python_home, $install_log) { + $install_args = "/quiet InstallAllUsers=1 TargetDir=$python_home" + RunCommand $exepath $install_args +} + + +function InstallPythonMSI ($msipath, $python_home, $install_log) { + $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home" + $uninstall_args = "/qn /x $msipath" + RunCommand "msiexec.exe" $install_args + if (-not(Test-Path $python_home)) { + Write-Host "Python seems to be installed else-where, reinstalling." + RunCommand "msiexec.exe" $uninstall_args + RunCommand "msiexec.exe" $install_args + } +} + +function RunCommand ($command, $command_args) { + Write-Host $command $command_args + Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru +} + + +function InstallPip ($python_home) { + $pip_path = $python_home + "\Scripts\pip.exe" + $python_path = $python_home + "\python.exe" + if (-not(Test-Path $pip_path)) { + Write-Host "Installing pip..." + $webclient = New-Object System.Net.WebClient + $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) + Write-Host "Executing:" $python_path $GET_PIP_PATH + & $python_path $GET_PIP_PATH + } else { + Write-Host "pip already installed." + } +} + + +function DownloadMiniconda ($python_version, $platform_suffix) { + if ($python_version -eq "3.4") { + $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe" + } else { + $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe" + } + $url = $MINICONDA_URL + $filename + $filepath = Download $filename $url + return $filepath +} + + +function InstallMiniconda ($python_version, $architecture, $python_home) { + Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home + if (Test-Path $python_home) { + Write-Host $python_home "already exists, skipping." + return $false + } + if ($architecture -eq "32") { + $platform_suffix = "x86" + } else { + $platform_suffix = "x86_64" + } + $filepath = DownloadMiniconda $python_version $platform_suffix + Write-Host "Installing" $filepath "to" $python_home + $install_log = $python_home + ".log" + $args = "/S /D=$python_home" + Write-Host $filepath $args + Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru + if (Test-Path $python_home) { + Write-Host "Python $python_version ($architecture) installation complete" + } else { + Write-Host "Failed to install Python in $python_home" + Get-Content -Path $install_log + Exit 1 + } +} + + +function InstallMinicondaPip ($python_home) { + $pip_path = $python_home + "\Scripts\pip.exe" + $conda_path = $python_home + "\Scripts\conda.exe" + if (-not(Test-Path $pip_path)) { + Write-Host "Installing pip..." + $args = "install --yes pip" + Write-Host $conda_path $args + Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru + } else { + Write-Host "pip already installed." + } +} + +function main () { + InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON + InstallPip $env:PYTHON +} + +main diff --git a/build_tools/appveyor/requirements.txt b/build_tools/appveyor/requirements.txt new file mode 100644 index 0000000..00a75fd --- /dev/null +++ b/build_tools/appveyor/requirements.txt @@ -0,0 +1,15 @@ +# Fetch numpy and scipy wheels from the sklearn rackspace wheelhouse. +# Those wheels were collected from http://www.lfd.uci.edu/~gohlke/pythonlibs/ +# This is a temporary solution. As soon as numpy and scipy provide official +# wheel for windows we ca delete this --find-links line. +--find-links http://28daf2247a33ed269873-7b1aad3fab3cc330e1fd9d109892382a.r6.cf2.rackcdn.com/ + +# fix the versions of numpy to force the use of numpy and scipy to use the whl +# of the rackspace folder instead of trying to install from more recent +# source tarball published on PyPI +numpy==1.9.3 +scipy==0.16.0 +sklearn==0.15.0 +nose +wheel +wheelhouse_uploader diff --git a/build_tools/appveyor/run_with_env.cmd b/build_tools/appveyor/run_with_env.cmd new file mode 100644 index 0000000..5da547c --- /dev/null +++ b/build_tools/appveyor/run_with_env.cmd @@ -0,0 +1,88 @@ +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) +:: +:: To build extensions for 64 bit Python 2, we need to configure environment +:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) +:: +:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific +:: environment configurations. +:: +:: Note: this script needs to be run with the /E:ON and /V:ON flags for the +:: cmd interpreter, at least for (SDK v7.0) +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows +:: http://stackoverflow.com/a/13751649/163740 +:: +:: Author: Olivier Grisel +:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +:: +:: Notes about batch files for Python people: +:: +:: Quotes in values are literally part of the values: +:: SET FOO="bar" +:: FOO is now five characters long: " b a r " +:: If you don't want quotes, don't include them on the right-hand side. +:: +:: The CALL lines at the end of this file look redundant, but if you move them +:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y +:: case, I don't know why. +@ECHO OFF + +SET COMMAND_TO_RUN=%* +SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows +SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf + +:: Extract the major and minor versions, and allow for the minor version to be +:: more than 9. This requires the version number to have two dots in it. +SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1% +IF "%PYTHON_VERSION:~3,1%" == "." ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1% +) ELSE ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2% +) + +:: Based on the Python version, determine what SDK version to use, and whether +:: to set the SDK for 64-bit. +IF %MAJOR_PYTHON_VERSION% == 2 ( + SET WINDOWS_SDK_VERSION="v7.0" + SET SET_SDK_64=Y +) ELSE ( + IF %MAJOR_PYTHON_VERSION% == 3 ( + SET WINDOWS_SDK_VERSION="v7.1" + IF %MINOR_PYTHON_VERSION% LEQ 4 ( + SET SET_SDK_64=Y + ) ELSE ( + SET SET_SDK_64=N + IF EXIST "%WIN_WDK%" ( + :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/ + REN "%WIN_WDK%" 0wdf + ) + ) + ) ELSE ( + ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" + EXIT 1 + ) +) + +IF %PYTHON_ARCH% == 64 ( + IF %SET_SDK_64% == Y ( + ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture + SET DISTUTILS_USE_SDK=1 + SET MSSdk=1 + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) ELSE ( + ECHO Using default MSVC build environment for 64 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) +) ELSE ( + ECHO Using default MSVC build environment for 32 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) diff --git a/build_tools/circle/build_doc.sh b/build_tools/circle/build_doc.sh new file mode 100755 index 0000000..bde8539 --- /dev/null +++ b/build_tools/circle/build_doc.sh @@ -0,0 +1,55 @@ +set -x +set -e + +# Introspect the commit to know whether or not we should skip building the +# documentation: a pull request that does not change any file in doc/ or +# examples/ folder should be skipped unless the "[doc: build]" is found the +# commit message. +BUILD_DOC=`python build_tools/circle/check_build_doc.py` +echo -e $BUILD_DOC +if [[ $BUILD_DOC == "SKIP:"* ]]; then + touch ~/log.txt # the "test" segment needs that file + exit 0 +fi + +# Installing required system packages to support the rendering of match +# notation in the HTML documentation +sudo -E apt-get -yq update +sudo -E apt-get -yq remove texlive-binaries --purge +sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes \ + install dvipng texlive-latex-base texlive-latex-extra + +# deactivate circleci virtualenv and setup a miniconda env instead +if [[ `type -t deactivate` ]]; then + deactivate +fi + +# Install dependencies with miniconda +pushd . +cd +mkdir -p download +cd download +echo "Cached in $HOME/download :" +ls -l +if [[ ! -f miniconda.sh ]] +then + wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh \ + -O miniconda.sh +fi +chmod +x miniconda.sh && ./miniconda.sh -b -p $HOME/miniconda +cd .. +export PATH="$HOME/miniconda/bin:$PATH" +conda update --yes --quiet conda +popd + +# Configure the conda environment and put it in the path using the +# provided versions +conda create -n testenv --yes --quiet python numpy scipy \ + cython nose coverage matplotlib sphinx pillow +source /home/ubuntu/miniconda/envs/testenv/bin/activate testenv + +# Build and install scikit-learn in dev mode +python setup.py develop + +# The pipefail is requested to propagate exit code +set -o pipefail && cd doc && make html 2>&1 | tee ~/log.txt diff --git a/build_tools/circle/check_build_doc.py b/build_tools/circle/check_build_doc.py new file mode 100644 index 0000000..f8e0048 --- /dev/null +++ b/build_tools/circle/check_build_doc.py @@ -0,0 +1,66 @@ +"""Check whether we or not we should build the documentation + +If the last commit message has a "[doc skip]" marker, do not build +the doc. On the contrary if a "[doc build]" marker is found, build the doc +instead of relying on the subsequent rules. + +We always build the documentation for jobs that are not related to a specific +PR (e.g. a merge to master or a maintenance branch). + +If this is a PR, check that if there are some files in this PR that are under +the "doc/" or "examples/" folders, otherwise skip. + +If the introspection of the current commit fails for any reason, the default +behavior is to build the documentation. + +""" +import sys +import os +from subprocess import check_output, CalledProcessError + + +def exit(msg="", skip=False): + print("%s: %s" % ("SKIP" if skip else "BUILD", msg)) + sys.exit(0) + +# Introspect the message for the commit that triggered the build +commit = os.environ.get('CIRCLE_SHA1') +if not commit: + exit("undefined CIRCLE_SHA1 variable") +try: + commit_msg = check_output("git log --format=%B -n 1".split() + [commit]) + commit_msg = commit_msg.decode('utf-8') +except CalledProcessError: + exit("failed to introspect commit message for %s" % commit) + +if "[doc skip]" in commit_msg: + exit("[doc skip] marker found", skip=True) +elif "[doc build]" in commit_msg: + exit("[doc build] marker found") + +# Check whether this commit is part of a pull request or not +pr_url = os.environ.get('CI_PULL_REQUEST') +if not pr_url: + # The documentation should be always built when executed from one of the + # main branches + exit("not a pull request") + +# Introspect the list of files changed by all the commits in this PR. +# Hardcode the assumption that this is a PR to origin/master of this repo +# as apparently there is way to reliably get the target of a PR with circle +# ci +git_range = "origin/master...%s" % commit +try: + check_output("git fetch origin master".split()) + filenames = check_output("git diff --name-only".split() + [git_range]) +except CalledProcessError: + exit("git introspection failed.") +filenames = filenames.decode('utf-8').split() +for filename in filenames: + if filename.startswith(u'doc/') or filename.startswith(u'examples/'): + exit("detected doc impacting file modified by PR in range %s: %s" + % (git_range, filename)) + +# This PR does not seem to have any documentation related file changed. +msg = "no doc impacting files detected:\n" + u"\n".join(filenames) +exit(msg, skip=True) diff --git a/build_tools/circle/push_doc.sh b/build_tools/circle/push_doc.sh new file mode 100755 index 0000000..2423929 --- /dev/null +++ b/build_tools/circle/push_doc.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# This script is meant to be called in the "deploy" step defined in +# circle.yml. See https://circleci.com/docs/ for more details. +# The behavior of the script is controlled by environment variable defined +# in the circle.yml in the top level folder of the project. + + +if [ -z $CIRCLE_PROJECT_USERNAME ]; +then USERNAME="sklearn-ci"; +else USERNAME=$CIRCLE_PROJECT_USERNAME; +fi + +DOC_REPO="scikit-learn.github.io" + +MSG="Pushing the docs for revision for branch: $CIRCLE_BRANCH, commit $CIRCLE_SHA1" + +cd $HOME +if [ ! -d $DOC_REPO ]; +then git clone "git@github.com:scikit-learn/"$DOC_REPO".git"; +fi +cd $DOC_REPO +git checkout master +git reset --hard origin/master +git rm -rf dev/ && rm -rf dev/ +cp -R $HOME/scikit-learn/doc/_build/html/stable dev +git config --global user.email "olivier.grisel+sklearn-ci@gmail.com" +git config --global user.name $USERNAME +git config --global push.default matching +git add -f dev/ +git commit -m "$MSG" dev +git push + +echo $MSG diff --git a/build_tools/cythonize.py b/build_tools/cythonize.py new file mode 100755 index 0000000..b01da58 --- /dev/null +++ b/build_tools/cythonize.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python +""" cythonize + +Cythonize pyx files into C files as needed. + +Usage: cythonize [root_dir] + +Default [root_dir] is 'sklearn'. + +Checks pyx files to see if they have been changed relative to their +corresponding C files. If they have, then runs cython on these files to +recreate the C files. + +The script detects changes in the pyx/pxd files using checksums +[or hashes] stored in a database file + +Simple script to invoke Cython on all .pyx +files; while waiting for a proper build system. Uses file hashes to +figure out if rebuild is needed. + +It is called by ./setup.py sdist so that sdist package can be installed without +cython + +Originally written by Dag Sverre Seljebotn, and adapted from statsmodel 0.6.1 +(Modified BSD 3-clause) + +We copied it for scikit-learn. + +Note: this script does not check any of the dependent C libraries; it only +operates on the Cython .pyx files or their corresponding Cython header (.pxd) +files. +""" +# Author: Arthur Mensch +# Author: Raghav R V +# +# License: BSD 3 clause + +from __future__ import division, print_function, absolute_import + +import os +import re +import sys +import hashlib +import subprocess + +HASH_FILE = 'cythonize.dat' +DEFAULT_ROOT = 'sklearn' + +# WindowsError is not defined on unix systems +try: + WindowsError +except NameError: + WindowsError = None + + +def cythonize(cython_file, gen_file): + try: + from Cython.Compiler.Version import version as cython_version + from distutils.version import LooseVersion + if LooseVersion(cython_version) < LooseVersion('0.21'): + raise Exception('Building scikit-learn requires Cython >= 0.21') + + except ImportError: + pass + + flags = ['--fast-fail'] + if gen_file.endswith('.cpp'): + flags += ['--cplus'] + + try: + try: + rc = subprocess.call(['cython'] + + flags + ["-o", gen_file, cython_file]) + if rc != 0: + raise Exception('Cythonizing %s failed' % cython_file) + except OSError: + # There are ways of installing Cython that don't result in a cython + # executable on the path, see scipy issue gh-2397. + rc = subprocess.call([sys.executable, '-c', + 'import sys; from Cython.Compiler.Main ' + 'import setuptools_main as main;' + ' sys.exit(main())'] + flags + + ["-o", gen_file, cython_file]) + if rc != 0: + raise Exception('Cythonizing %s failed' % cython_file) + except OSError: + raise OSError('Cython needs to be installed') + + +def load_hashes(filename): + """Load the hashes dict from the hashfile""" + # { filename : (sha1 of header if available or 'NA', + # sha1 of input, + # sha1 of output) } + + hashes = {} + try: + with open(filename, 'r') as cython_hash_file: + for hash_record in cython_hash_file: + (filename, header_hash, + cython_hash, gen_file_hash) = hash_record.split() + hashes[filename] = (header_hash, cython_hash, gen_file_hash) + except (KeyError, ValueError, AttributeError, IOError): + hashes = {} + return hashes + + +def save_hashes(hashes, filename): + """Save the hashes dict to the hashfile""" + with open(filename, 'w') as cython_hash_file: + for key, value in hashes.items(): + cython_hash_file.write("%s %s %s %s\n" + % (key, value[0], value[1], value[2])) + + +def sha1_of_file(filename): + h = hashlib.sha1() + with open(filename, "rb") as f: + h.update(f.read()) + return h.hexdigest() + + +def clean_path(path): + """Clean the path""" + path = path.replace(os.sep, '/') + if path.startswith('./'): + path = path[2:] + return path + + +def get_hash_tuple(header_path, cython_path, gen_file_path): + """Get the hashes from the given files""" + + header_hash = (sha1_of_file(header_path) + if os.path.exists(header_path) else 'NA') + from_hash = sha1_of_file(cython_path) + to_hash = (sha1_of_file(gen_file_path) + if os.path.exists(gen_file_path) else 'NA') + + return header_hash, from_hash, to_hash + + +def cythonize_if_unchanged(path, cython_file, gen_file, hashes): + full_cython_path = os.path.join(path, cython_file) + full_header_path = full_cython_path.replace('.pyx', '.pxd') + full_gen_file_path = os.path.join(path, gen_file) + + current_hash = get_hash_tuple(full_header_path, full_cython_path, + full_gen_file_path) + + if current_hash == hashes.get(clean_path(full_cython_path)): + print('%s has not changed' % full_cython_path) + return + + print('Processing %s' % full_cython_path) + cythonize(full_cython_path, full_gen_file_path) + + # changed target file, recompute hash + current_hash = get_hash_tuple(full_header_path, full_cython_path, + full_gen_file_path) + + # Update the hashes dict with the new hash + hashes[clean_path(full_cython_path)] = current_hash + + +def check_and_cythonize(root_dir): + print(root_dir) + hashes = load_hashes(HASH_FILE) + + for cur_dir, dirs, files in os.walk(root_dir): + for filename in files: + if filename.endswith('.pyx'): + gen_file_ext = '.c' + # Cython files with libcpp imports should be compiled to cpp + with open(os.path.join(cur_dir, filename), 'rb') as f: + data = f.read() + m = re.search(b"libcpp", data, re.I | re.M) + if m: + gen_file_ext = ".cpp" + cython_file = filename + gen_file = filename.replace('.pyx', gen_file_ext) + cythonize_if_unchanged(cur_dir, cython_file, gen_file, hashes) + + # Save hashes once per module. This prevents cythonizing prev. + # files again when debugging broken code in a single file + save_hashes(hashes, HASH_FILE) + + +def main(root_dir=DEFAULT_ROOT): + check_and_cythonize(root_dir) + + +if __name__ == '__main__': + try: + root_dir_arg = sys.argv[1] + except IndexError: + root_dir_arg = DEFAULT_ROOT + main(root_dir_arg) diff --git a/build_tools/travis/after_success.sh b/build_tools/travis/after_success.sh new file mode 100755 index 0000000..a4613cc --- /dev/null +++ b/build_tools/travis/after_success.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# This script is meant to be called by the "after_success" step defined in +# .travis.yml. See http://docs.travis-ci.com/ for more details. + +# License: 3-clause BSD + +set -e + +if [[ "$COVERAGE" == "true" ]]; then + # Need to run coveralls from a git checkout, so we copy .coverage + # from TEST_DIR where nosetests has been run + cp $TEST_DIR/.coverage $TRAVIS_BUILD_DIR + cd $TRAVIS_BUILD_DIR + # Ignore coveralls failures as the coveralls server is not + # very reliable but we don't want travis to report a failure + # in the github UI just because the coverage report failed to + # be published. + coveralls || echo "Coveralls upload failed" +fi diff --git a/build_tools/travis/install.sh b/build_tools/travis/install.sh new file mode 100755 index 0000000..a3f122b --- /dev/null +++ b/build_tools/travis/install.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# This script is meant to be called by the "install" step defined in +# .travis.yml. See http://docs.travis-ci.com/ for more details. +# The behavior of the script is controlled by environment variabled defined +# in the .travis.yml in the top level folder of the project. + +# License: 3-clause BSD + +# Travis clone scikit-learn/scikit-learn repository in to a local repository. +# We use a cached directory with three scikit-learn repositories (one for each +# matrix entry) from which we pull from local Travis repository. This allows +# us to keep build artefact for gcc + cython, and gain time + +set -e + +# Fix the compilers to workaround avoid having the Python 3.4 build +# lookup for g++44 unexpectedly. +export CC=gcc +export CXX=g++ + +echo 'List files from cached directories' +echo 'pip:' +ls $HOME/.cache/pip + + +if [[ "$DISTRIB" == "conda" ]]; then + # Deactivate the travis-provided virtual environment and setup a + # conda-based environment instead + deactivate + + # Use the miniconda installer for faster download / install of conda + # itself + pushd . + cd + mkdir -p download + cd download + echo "Cached in $HOME/download :" + ls -l + echo + if [[ ! -f miniconda.sh ]] + then + wget http://repo.continuum.io/miniconda/Miniconda-3.6.0-Linux-x86_64.sh \ + -O miniconda.sh + fi + chmod +x miniconda.sh && ./miniconda.sh -b + cd .. + export PATH=/home/travis/miniconda/bin:$PATH + conda update --yes conda + popd + + # Configure the conda environment and put it in the path using the + # provided versions + if [[ "$INSTALL_MKL" == "true" ]]; then + conda create -n testenv --yes python=$PYTHON_VERSION pip nose \ + numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION numpy scipy \ + cython=$CYTHON_VERSION libgfortran mkl + else + conda create -n testenv --yes python=$PYTHON_VERSION pip nose \ + numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION cython=$CYTHON_VERSION \ + libgfortran + fi + source activate testenv + + # Install nose-timer via pip + pip install nose-timer + +elif [[ "$DISTRIB" == "ubuntu" ]]; then + # At the time of writing numpy 1.9.1 is included in the travis + # virtualenv but we want to use the numpy installed through apt-get + # install. + deactivate + # Create a new virtualenv using system site packages for python, numpy + # and scipy + virtualenv --system-site-packages testvenv + source testvenv/bin/activate + pip install nose nose-timer cython + +elif [[ "$DISTRIB" == "scipy-dev-wheels" ]]; then + # Set up our own virtualenv environment to avoid travis' numpy. + # This venv points to the python interpreter of the travis build + # matrix. + virtualenv --python=python ~/testvenv + source ~/testvenv/bin/activate + pip install --upgrade pip setuptools + + # We use the default Python virtualenv provided by travis + echo "Installing numpy master wheel" + pip install --pre --upgrade --no-index --timeout=60 \ + --trusted-host travis-dev-wheels.scipy.org \ + -f https://travis-dev-wheels.scipy.org/ numpy scipy + pip install nose nose-timer cython +fi + +if [[ "$COVERAGE" == "true" ]]; then + pip install coverage coveralls +fi + +if [ ! -d "$CACHED_BUILD_DIR" ]; then + mkdir -p $CACHED_BUILD_DIR +fi + +rsync -av --exclude '.git/' --exclude='testvenv/' \ + $TRAVIS_BUILD_DIR $CACHED_BUILD_DIR + +cd $CACHED_BUILD_DIR/scikit-learn + +# Build scikit-learn in the install.sh script to collapse the verbose +# build output in the travis output when it succeeds. +python --version +python -c "import numpy; print('numpy %s' % numpy.__version__)" +python -c "import scipy; print('scipy %s' % scipy.__version__)" +python setup.py develop diff --git a/build_tools/travis/test_script.sh b/build_tools/travis/test_script.sh new file mode 100755 index 0000000..cad92e5 --- /dev/null +++ b/build_tools/travis/test_script.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# This script is meant to be called by the "script" step defined in +# .travis.yml. See http://docs.travis-ci.com/ for more details. +# The behavior of the script is controlled by environment variabled defined +# in the .travis.yml in the top level folder of the project. + +# License: 3-clause BSD + +set -e + +# Get into a temp directory to run test from the installed scikit learn and +# check if we do not leave artifacts +mkdir -p $TEST_DIR +# We need the setup.cfg for the nose settings +cp setup.cfg $TEST_DIR +cd $TEST_DIR + +python --version +python -c "import numpy; print('numpy %s' % numpy.__version__)" +python -c "import scipy; print('scipy %s' % scipy.__version__)" +python -c "import multiprocessing as mp; print('%d CPUs' % mp.cpu_count())" + +# Skip tests that require large downloads over the network to save bandwidth +# usage as travis workers are stateless and therefore traditional local +# disk caching does not work. +export SKLEARN_SKIP_NETWORK_TESTS=1 + +if [[ "$COVERAGE" == "true" ]]; then + nosetests -s --with-coverage --with-timer --timer-top-n 20 sklearn +else + nosetests -s --with-timer --timer-top-n 20 sklearn +fi + +# Is directory still empty ? +ls -ltra + +# Test doc +cd $CACHED_BUILD_DIR/scikit-learn +make test-doc test-sphinxext diff --git a/build_tools/windows/windows_testing_downloader.ps1 b/build_tools/windows/windows_testing_downloader.ps1 new file mode 100644 index 0000000..d72b678 --- /dev/null +++ b/build_tools/windows/windows_testing_downloader.ps1 @@ -0,0 +1,270 @@ +# Author: Kyle Kastner +# License: BSD 3 clause + +# This script is a helper to download the base python, numpy, and scipy +# packages from their respective websites. +# To quickly execute the script, run the following Powershell command: +# powershell.exe -ExecutionPolicy unrestricted "iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/scikit-learn/scikit-learn/master/continuous_integration/windows/windows_testing_downloader.ps1'))" + +# This is a stopgap solution to make Windows testing easier +# until Windows CI issues are resolved. + +# Rackspace's default Windows VMs have several security features enabled by default. +# The DisableInternetExplorerESC function disables a feature which +# prevents any webpage from opening without explicit permission. +# This is a default setting of Windows VMs on Rackspace, and makes it annoying to +# download other packages to test! + +# Powershell scripts are also disabled by default. One must run the command: +# set-executionpolicy unrestricted +# from a Powershell terminal with administrator rights to enable scripts. +# To start an administrator Powershell terminal, right click second icon from the left on Windows Server 2012's bottom taskbar. + +param ( + [string]$python = "None", + [string]$nogit = "False" +) + +function DisableInternetExplorerESC { + # Disables InternetExplorerESC to enable easier manual downloads of testing packages. + # http://stackoverflow.com/questions/9368305/disable-ie-security-on-windows-server-via-powershell + $AdminKey = "HKLM:\SOFTWARE\Microsoft\Active Setup\Installed Components\{A509B1A7-37EF-4b3f-8CFC-4F3A74704073}" + $UserKey = "HKLM:\SOFTWARE\Microsoft\Active Setup\Installed Components\{A509B1A8-37EF-4b3f-8CFC-4F3A74704073}" + Set-ItemProperty -Path $AdminKey -Name "IsInstalled" -Value 0 + Set-ItemProperty -Path $UserKey -Name "IsInstalled" -Value 0 + Stop-Process -Name Explorer + Write-Host "IE Enhanced Security Configuration (ESC) has been disabled." -ForegroundColor Green +} + +function DownloadPackages ($package_dict, $append_string) { + $webclient = New-Object System.Net.WebClient + + ForEach ($key in $package_dict.Keys) { + $url = $package_dict[$key] + $file = $key + $append_string + if ($url -match "(\.*exe$)") { + $file = $file + ".exe" + } elseif ($url -match "(\.*msi$)") { + $file = $file + ".msi" + } else { + $file = $file + ".py" + } + $basedir = $pwd.Path + "\" + $filepath = $basedir + $file + Write-Host "Downloading" $file "from" $url + + # Retry up to 5 times in case of network transient errors. + $retry_attempts = 5 + for($i=0; $i -lt $retry_attempts; $i++){ + try{ + $webclient.DownloadFile($url, $filepath) + break + } + Catch [Exception]{ + Start-Sleep 1 + } + } + Write-Host "File saved at" $filepath + } +} + +function InstallPython($match_string) { + $pkg_regex = "python" + $match_string + "*" + $pkg = Get-ChildItem -Filter $pkg_regex -Name + Invoke-Expression -Command "msiexec /qn /i $pkg" + + Write-Host "Installing Python" + Start-Sleep 25 + Write-Host "Python installation complete" +} + +function InstallPip($match_string, $python_version) { + $pkg_regex = "get-pip" + $match_string + "*" + $py = $python_version -replace "\." + $pkg = Get-ChildItem -Filter $pkg_regex -Name + $python_path = "C:\Python" + $py + "\python.exe" + Invoke-Expression -Command "$python_path $pkg" +} + +function EnsurePip($python_version) { + $py = $python_version -replace "\." + $python_path = "C:\Python" + $py + "\python.exe" + Invoke-Expression -Command "$python_path -m ensurepip" +} + +function GetPythonHome($python_version) { + $py = $python_version -replace "\." + $pypath = "C:\Python" + $py + "\" + return $pypath +} + +function GetPipPath($python_version) { + $py = $python_version -replace "\." + $pypath = GetPythonHome $python_version + if ($py.StartsWith("3")) { + $pip = $pypath + "Scripts\pip3.exe" + } else { + $pip = $pypath + "Scripts\pip.exe" + } + return $pip +} + +function PipInstall($pkg_name, $python_version, $extra_args) { + $pip = GetPipPath $python_version + Invoke-Expression -Command "$pip install $pkg_name" +} + +function InstallNose($python_version) { + PipInstall "nose" $python_version +} + +function WheelInstall($name, $url, $python_version) { + $pip = GetPipPath $python_version + $args = "install --use-wheel --no-index" + Invoke-Expression -Command "$pip $args $url $name" +} + +function InstallWheel($python_version) { + PipInstall "virtualenv" $python_version + PipInstall "wheel" $python_version +} + +function InstallNumpy($package_dict, $python_version) { + #Don't pass name so we can use URL directly. + WheelInstall "" $package_dict["numpy"] $python_version +} + +function InstallScipy($package_dict, $python_version) { + #Don't pass name so we can use URL directly. + WheelInstall "" $package_dict["scipy"] $python_version +} + +function InstallGit { + $pkg_regex = "git*" + $pkg = Get-ChildItem -Filter $pkg_regex -Name + $pkg_cmd = $pwd.ToString() + "\" + $pkg + " /verysilent" + Invoke-Expression -Command $pkg_cmd + + Write-Host "Installing Git" + Start-Sleep 20 + # Remove the installer - seems to cause weird issues with Git Bash + Invoke-Expression -Command "rm git.exe" + Write-Host "Git installation complete" +} + +function ReadAndUpdateFromRegistry { + # http://stackoverflow.com/questions/14381650/how-to-update-windows-powershell-session-environment-variables-from-registry + foreach($level in "Machine","User") { + [Environment]::GetEnvironmentVariables($level).GetEnumerator() | % { + # For Path variables, append the new values, if they're not already in there + if($_.Name -match 'Path$') { + $_.Value = ($((Get-Content "Env:$($_.Name)") + ";$($_.Value)") -split ';' | Select -unique) -join ';' + } + $_ + } | Set-Content -Path { "Env:$($_.Name)" } + } +} + +function UpdatePaths($python_version) { + #This function makes local path updates required in order to install Python and supplementary packages in a single shell. + $pypath = GetPythonHome $python_version + $env:PATH = $env:PATH + ";" + $pypath + $env:PYTHONPATH = $pypath + "DLLs;" + $pypath + "Lib;" + $pypath + "Lib\site-packages" + $env:PYTHONHOME = $pypath + Write-Host "PYTHONHOME temporarily set to" $env:PYTHONHOME + Write-Host "PYTHONPATH temporarily set to" $env:PYTHONPATH + Write-Host "PATH temporarily set to" $env:PATH +} + +function Python27URLs { + # Function returns a dictionary of packages to download for Python 2.7. + $urls = @{ + "python" = "https://www.python.org/ftp/python/2.7.7/python-2.7.7.msi" + "numpy" = "http://28daf2247a33ed269873-7b1aad3fab3cc330e1fd9d109892382a.r6.cf2.rackcdn.com/numpy-1.8.1-cp27-none-win32.whl" + "scipy" = "http://28daf2247a33ed269873-7b1aad3fab3cc330e1fd9d109892382a.r6.cf2.rackcdn.com/scipy-0.14.0-cp27-none-win32.whl" + "get-pip" = "https://bootstrap.pypa.io/get-pip.py" + } + return $urls +} + +function Python34URLs { + # Function returns a dictionary of packages to download for Python 3.4. + $urls = @{ + "python" = "https://www.python.org/ftp/python/3.4.1/python-3.4.1.msi" + "numpy" = "http://28daf2247a33ed269873-7b1aad3fab3cc330e1fd9d109892382a.r6.cf2.rackcdn.com/numpy-1.8.1-cp34-none-win32.whl" + "scipy" = "http://28daf2247a33ed269873-7b1aad3fab3cc330e1fd9d109892382a.r6.cf2.rackcdn.com/scipy-0.14.0-cp34-none-win32.whl" + } + return $urls +} + +function GitURLs { + # Function returns a dictionary of packages to download for Git + $urls = @{ + "git" = "https://github.com/msysgit/msysgit/releases/download/Git-1.9.4-preview20140611/Git-1.9.4-preview20140611.exe" + } + return $urls +} + +function main { + $versions = @{ + "2.7" = Python27URLs + "3.4" = Python34URLs + } + + if ($nogit -eq "False") { + Write-Host "Downloading and installing Gitbash" + $urls = GitURLs + DownloadPackages $urls "" + InstallGit ".exe" + } + + if (($python -eq "None")) { + Write-Host "Installing all supported python versions" + Write-Host "Current versions supported are:" + ForEach ($key in $versions.Keys) { + Write-Host $key + $all_python += @($key) + } + } elseif(!($versions.ContainsKey($python))) { + Write-Host "Python version not recognized!" + Write-Host "Pass python version with -python" + Write-Host "Current versions supported are:" + ForEach ($key in $versions.Keys) { + Write-Host $key + } + return + } else { + $all_python += @($python) + } + ForEach ($py in $all_python) { + Write-Host "Installing Python" $py + DisableInternetExplorerESC + $pystring = $py -replace "\." + $pystring = "_py" + $pystring + $package_dict = $versions[$py] + + # This will download the whl packages as well which is + # clunky but makes configuration simpler. + DownloadPackages $package_dict $pystring + UpdatePaths $py + InstallPython $pystring + ReadAndUpdateFromRegistry + if ($package_dict.ContainsKey("get-pip")) { + InstallPip $pystring $py + } else { + EnsurePip $py + } + InstallNose $py + InstallWheel $py + + # The installers below here use wheel packages. + # Wheels were created from CGohlke's installers with + # wheel convert + # These are hosted in Rackspace Cloud Files. + InstallNumpy $package_dict $py + InstallScipy $package_dict $py + } + return +} + +main