Merge pull request #225 from giotto-ai/master

Fix Windows PyPI distribution woes due to direct reference specifications

.azure-ci/python-igraph_dependencies_win.py

@@ -0,0 +1,28 @@
+import sys
+
+
+def whl_urls(python_ver, pkg):
+    if python_ver == '38':
+        python_ver_1 = python_ver
+    else:
+        python_ver_1 = python_ver + 'm'
+    pycairo_whl_url = \
+        'https://storage.googleapis.com/l2f-open-models/giotto' \
+        '-learn/windows-binaries/pycairo/pycairo-1.18.2-cp{}' \
+        '-cp{}-win_amd64.whl'.format(python_ver, python_ver_1)
+    igraph_whl_url = \
+        'https://storage.googleapis.com/l2f-open-models/giotto' \
+        '-learn/windows-binaries/python-igraph/python_igraph-' \
+        '0.7.1.post6-cp{}-cp{}-win_amd64.whl'.\
+        format(python_ver, python_ver_1)
+    if pkg == 'pycairo':
+        print(pycairo_whl_url)
+    elif pkg == 'python-igraph':
+        print(igraph_whl_url)
+    else:
+        raise ValueError("Second argument must be either 'pycairo' or "
+                         "python-igraph")
+
+
+if __name__ == '__main__':
+    whl_urls(sys.argv[1], sys.argv[2])

README.rst

@@ -74,21 +74,41 @@ To run the examples, jupyter is required.
 User installation
 ~~~~~~~~~~~~~~~~~
 
+Linux and macOS
+'''''''''''''''
 The simplest way to install giotto-tda is using ``pip``   ::
 
     pip install -U giotto-tda
 
-Note: the above may fail on old versions of ``pip``. We recommend upgrading ``pip``
-to a recent version.
+If necessary, this will also automatically install all the above dependencies. Note: we recommend
+upgrading ``pip`` to a recent version as the above may fail on very old versions.
 
 Pre-release, experimental builds containing recently added features, and/or
 bug fixes can be installed by running   ::
 
     pip install -U giotto-tda-nightly
 
-The main difference between ``giotto-tda-nightly`` and the developer
-installation (see below) is that the former is shipped with pre-compiled wheels
-(similarly to the stable release) and hence does not require any C++ dependencies.
+The main difference between giotto-tda-nightly and the developer installation (see below)
+is that the former is shipped with pre-compiled wheels (similarly to the stable release)
+and hence does not require any C++ dependencies.
+
+Windows
+'''''''
+In this case, python-igraph and its dependency pycairo must be manually installed before
+proceeding as above. This is because the python-igraph project does not yet provide official
+installers for Windows via PyPI, so that ``pip install python-igraph`` would fail there.
+The preferred way to install python-igraph on Windows is to download and install the relevant
+wheels built by Christoph Gohlke for both `pycairo <https://www.lfd.uci.edu/~gohlke/pythonlibs/#pycairo>`_
+and `python-igraph <https://www.lfd.uci.edu/~gohlke/pythonlibs/#python-igraph>`_. We
+host these wheels so they can be fetched with convenient URLs. For Python 3.5 to 3.7, you may run   ::
+
+    pip install https://storage.googleapis.com/l2f-open-models/giotto-learn/windows-binaries/pycairo/pycairo-1.18.2-cp<PYTHON VERSION>-cp<PYTHON VERSION>m-win_amd64.whl
+    pip install https://storage.googleapis.com/l2f-open-models/giotto-learn/windows-binaries/python-igraph/python_igraph-0.7.1.post6-cp<PYTHON VERSION>-cp<PYTHON VERSION>m-win_amd64.whl
+
+where ``<PYTHON VERSION>`` is e.g. ``37`` for Python 3.7. For Python 3.8, you may run   ::
+
+    pip install https://storage.googleapis.com/l2f-open-models/giotto-learn/windows-binaries/pycairo/pycairo-1.18.2-cp<PYTHON VERSION>-cp<PYTHON VERSION>-win_amd64.whl
+    pip install https://storage.googleapis.com/l2f-open-models/giotto-learn/windows-binaries/python-igraph/python_igraph-0.7.1.post6-cp<PYTHON VERSION>-cp<PYTHON VERSION>-win_amd64.whl
 
 Contributing
 ------------
@@ -102,7 +122,8 @@ Developer installation
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 Installing both the PyPI release and source of giotto-tda in the same environment is not recommended since it is
-known to cause conflicts with the C++ bindings.
+known to cause conflicts with the C++ bindings. On  Windows, the pycairo and python-igraph dependencies have to be
+installed manually just as in the case of a simple user installation.
 
 The developer installation requires three important C++ dependencies:
 

RELEASE.rst

@@ -182,11 +182,14 @@ Thanks to our Contributors
 
 This release contains contributions from many people:
 
-Guillaume Tauzin, Umberto Lupo, Philippe Nguyen, Matteo Caorsi, Julian Burella Pérez,
-Alessio Ghiraldello.
+Guillaume Tauzin, Umberto Lupo, Philippe Nguyen, Matteo Caorsi, Julian Burella Pérez, Alessio Ghiraldello.
 
 We are also grateful to all who filed issues or helped resolve them, asked and
-answered questions, and were part of inspiring discussions.
+answered questions, and were part of inspiring discussions. In particular, we would like
+to thank `Martino Milani <https://github.com/MartMilani/reportPACS>`_, who worked on an early
+prototype of a Mapper implementation; although very different from the current one, it
+adopted an early form of caching to avoid recomputation in refitting, which was an inspiration
+for this implementation.
 
 
 Release 0.1a.0

azure-pipelines.yml

@@ -108,7 +108,14 @@ jobs:
       pip install wheel twine
     displayName: 'Install tools'
 
-  - script: pip install -e ".[tests, doc]"
+  - bash: |
+      pycairo_url=$(python .azure-ci/python-igraph_dependencies_win.py '$(python_ver)' 'pycairo')
+      igraph_url=$(python .azure-ci/python-igraph_dependencies_win.py '$(python_ver)' 'python-igraph')
+      pip install "$pycairo_url" "$igraph_url"
+    displayName: 'Install pycairo and python-igraph'
+
+  - script: |
+      pip install -e ".[tests, doc]"
     displayName: 'Install dev environment'
 
   - script: |
@@ -118,7 +125,8 @@ jobs:
       flake8
     displayName: 'Test with pytest, nbconvert and flake8'
 
-# these jobs are triggered manually and they test the code and the examples and build the wheels and docs.
+
+# These jobs are triggered manually and they test the code and the examples and build the wheels and docs.
 
 - job: 'manylinux2010'
   condition: eq(variables['build_check'], 'true')
@@ -157,7 +165,7 @@ jobs:
       sed -i "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" gtda/_version.py
       cat gtda/_version.py
     condition: eq(variables['nightly_check'], 'true')
-    displayName: 'change name to giotto-tda-nightly'
+    displayName: 'Change name to giotto-tda-nightly'
 
   - task: Bash@3
     inputs:
@@ -179,13 +187,13 @@ jobs:
     displayName: 'Install and test the wheels'
 
   - task: CopyFiles@2
-    displayName: 'copy files'
+    displayName: 'Copy files'
     inputs:
       contents: 'dist/*'
       targetFolder: '$(Build.ArtifactStagingDirectory)'
 
   - task: PublishBuildArtifacts@1
-    displayName: 'create download link'
+    displayName: 'Create download link'
     inputs:
       pathToPublish: '$(Build.ArtifactStagingDirectory)'
       artifactName: 'wheel_and_doc'
@@ -225,13 +233,13 @@ jobs:
       cat gtda/_version.py
       rm gtda/_version.py.bak
     condition: eq(variables['nightly_check'], 'true')
-    displayName: 'change name to giotto-tda-nightly'
+    displayName: 'Change name to giotto-tda-nightly'
 
   - script: |
       brew update
       brew install boost
       brew install gcc
-    displayName: 'install boost and gcc'
+    displayName: 'Install boost and gcc'
 
   - script: python -m pip install --upgrade pip setuptools
     displayName: 'Install tools'
@@ -260,13 +268,13 @@ jobs:
     displayName: 'Test with pytest, nbconvert and flake8'
 
   - script: python setup.py sdist bdist_wheel
-    displayName: 'build the wheels'
+    displayName: 'Build the wheels'
 
   - script: |
       pip install dist/*.whl
       cd /tmp/
       pytest --cov --pyargs gtda --cov-report xml --ignore-glob='*externals*'
-    displayName: 'install and test the wheels'
+    displayName: 'Install and test the wheels'
 
   - script: |
       cd doc/
@@ -282,16 +290,16 @@ jobs:
       tarCompression: 'gz'
       archiveFile: '$(Build.ArtifactStagingDirectory)/$(Build.BuildId).zip'
       replaceExistingArchive: true
-    displayName: 'archive doc'
+    displayName: 'Archive doc'
 
   - task: CopyFiles@2
-    displayName: 'copy files'
+    displayName: 'Copy files'
     inputs:
       contents: 'dist/*'
       targetFolder: '$(Build.ArtifactStagingDirectory)'
 
   - task: PublishBuildArtifacts@1
-    displayName: 'create download link'
+    displayName: 'Create download link'
     inputs:
       pathToPublish: '$(Build.ArtifactStagingDirectory)'
       artifactName: 'wheel_and_doc'
@@ -326,15 +334,21 @@ jobs:
       versionSpec: '$(python.version)'
 
   - bash: |
-      sed -i "s/'giotto-tda'/'giotto-tda'/1" setup.py
+      sed -i "s/'giotto-tda'/'giotto-tda-nightly'/1" setup.py
       sed -i "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" gtda/_version.py
       cat gtda/_version.py
     condition: eq(variables['nightly_check'], 'true')
-    displayName: 'change name to giotto-tda-nightly'
+    displayName: 'Change name to giotto-tda-nightly'
 
   - script: python -m pip install --upgrade pip setuptools
     displayName: 'Install tools'
 
+  - bash: |
+      pycairo_url=$(python .azure-ci/python-igraph_dependencies_win.py '$(python_ver)' 'pycairo')
+      igraph_url=$(python .azure-ci/python-igraph_dependencies_win.py '$(python_ver)' 'python-igraph')
+      pip install "$pycairo_url" "$igraph_url"
+    displayName: 'Install pycairo and python-igraph'
+
   - script: |
       pip install -e ".[tests, doc]"
       pip install wheel twine
@@ -361,22 +375,22 @@ jobs:
   - bash: |
       sed -i $'s/\r$//' README.rst
       python setup.py sdist bdist_wheel
-    displayName: 'build the wheels'
+    displayName: 'Build the wheels'
 
   - bash: |
       pip install dist/*.whl
       cd /tmp/
       pytest --cov --pyargs gtda --cov-report xml --ignore-glob='*externals*'
-    displayName: 'install and test the wheels'
+    displayName: 'Install and test the wheels'
 
   - task: CopyFiles@2
-    displayName: 'copy files'
+    displayName: 'Copy files'
     inputs:
       contents: 'dist/*'
       targetFolder: '$(Build.ArtifactStagingDirectory)'
 
   - task: PublishBuildArtifacts@1
-    displayName: 'create download link'
+    displayName: 'Create download link'
     inputs:
       pathToPublish: '$(Build.ArtifactStagingDirectory)'
       artifactName: 'wheel_and_doc'

gtda/mapper/pipeline.py

@@ -155,10 +155,10 @@ def make_mapper_pipeline(scaler=None,
     steps. [1]_
 
     The role of this function's key parameters is illustrated in `this diagram
-    <https://docs-tda.giotto.ai/mapper_pipeline.svg>`_. All computational steps
-    may be arbitrary scikit-learn Pipeline objects. The scaling and cover
-    steps must be transformers implementing a ``fit_transform`` method. The
-    filter function step may be a transformer implementing a ``fit_transform``,
+    </mapper_pipeline.svg>`_. All computational steps may be arbitrary
+    scikit-learn Pipeline objects. The scaling and cover steps must be
+    transformers implementing a ``fit_transform`` method. The filter
+    function step may be a transformer implementing a ``fit_transform``,
     or a callable acting on one-dimensional arrays -- in the latter case,
     a transformer is internally created whose ``fit_transform`` applies this
     callable independently on each row of the data. The clustering step need
@@ -179,9 +179,10 @@ def make_mapper_pipeline(scaler=None,
 
     clustering_preprocessing : object or None, optional, default: ``None``
         If not ``None``, it is a transformer which is applied to the
-        data independently to the `scaler` -> `filter_func` -> cover` pipeline.
-        Clustering is then performed on portions (determined by the `scaler`
-        -> `filter_func` -> cover` pipeline) of the transformed data.
+        data independently to the `scaler` -> `filter_func` -> `cover`
+        pipeline. Clustering is then performed on portions (determined by
+        the `scaler` -> `filter_func` -> `cover` pipeline) of the
+        transformed data.
 
     clusterer : object or None, optional, default: ``None``
         Clustering object. ``None`` means using DBSCAN

requirements.txt

@@ -2,6 +2,7 @@ numpy >= 1.17.0
 scipy >= 0.17.0
 joblib >= 0.11
 scikit-learn >= 0.22.0
+python-igraph >= 0.7.1.post6
 matplotlib >= 3.0.3
 plotly >= 4.4.1
 ipywidgets >= 7.5.1

setup.py

@@ -49,27 +49,6 @@
 KEYWORDS = 'machine learning, topological data analysis, persistent ' + \
     'homology, persistence diagrams, Mapper'
 INSTALL_REQUIRES = requirements
-is_system_win = platform.system() == 'Windows'
-if is_system_win:
-    python_ver = sys.version_info
-    python_ver_1 = str(python_ver.major) + str(python_ver.minor)
-    if python_ver_1 == '38':
-        python_ver_2 = python_ver_1
-    else:
-        python_ver_2 = python_ver_1 + 'm'
-    pycairo_whl_url = \
-        'https://storage.googleapis.com/l2f-open-models/giotto' \
-        '-learn/windows-binaries/pycairo/pycairo-1.18.2-cp{}' \
-        '-cp{}-win_amd64.whl'.format(python_ver_1, python_ver_2)
-    igraph_whl_url = \
-        'https://storage.googleapis.com/l2f-open-models/giotto' \
-        '-learn/windows-binaries/python-igraph/python_igraph-' \
-        '0.7.1.post6-cp{}-cp{}-win_amd64.whl'.\
-        format(python_ver_1, python_ver_2)
-    INSTALL_REQUIRES.append('pycairo @ {}'.format(pycairo_whl_url))
-    INSTALL_REQUIRES.append('python-igraph @ {}'.format(igraph_whl_url))
-else:
-    INSTALL_REQUIRES.append('python-igraph')
 EXTRAS_REQUIRE = {
     'tests': [
         'pytest',