Merge pull request #27 from gmrukwa/develop

Release 2.3.8

.dockerignore

@@ -0,0 +1,112 @@
+# Git
+.git/
+
+# PyCharm
+.idea/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# local tests
+data.csv
+xy.csv
+result

.github/actions/build-python-dist/Dockerfile

@@ -1,4 +1,8 @@
-FROM spectreteam/python_msi:v5.1.0.2019a.py37
+FROM python:3.7-slim
+
+RUN apt-get update &&\
+    apt-get install -y gcc &&\
+    rm -rf /var/lib/apt/lists/*
 
 RUN pip install --no-cache-dir --upgrade \
     setuptools \

.github/actions/build-python-dist/entrypoint.sh

@@ -2,6 +2,6 @@
 
 pip install --no-cache-dir -r requirements.txt
 
-python setup.py sdist bdist_wheel
+python setup.py sdist
 
 ls dist

.github/workflows/deploy.yml

@@ -12,7 +12,7 @@ on:
 env:
   MAJOR: ${{ 2 }}
   MINOR: ${{ 3 }}
-  FIXUP: ${{ 6 }}
+  FIXUP: ${{ 8 }}
   PACKAGE_INIT_FILE: ${{ 'divik/__init__.py' }}
   DOCKER_REPO: ${{ 'gmrukwa/divik' }}
   IS_ALPHA: ${{ github.event_name == 'pull_request' }}

README.md

@@ -28,8 +28,7 @@ dataset
 
 The recommended way to use this software is through
 [Docker](https://www.docker.com/). This is the most convenient way, if you want
-to use `divik` application, since it requires *MATLAB Compiler Runtime*
-and more dependencies.
+to use `divik` application.
 
 To install latest stable version use:
 
@@ -40,30 +39,15 @@ docker pull gmrukwa/divik
 To install specific version, you can specify it in the command, e.g.:
 
 ```bash
-docker pull gmrukwa/divik:2.3.6
+docker pull gmrukwa/divik:2.3.8
 ```
 
 ## Python package
 
 Prerequisites for installation of base package:
 
 - Python 3.5 / 3.6 / 3.7
-
-These are required for using `divik` application and GMM-based filtering:
-
-- [MATLAB Compiler Runtime](https://www.mathworks.com/products/compiler/matlab-runtime.html),
-version 2016b or newer, installed to default path
-- [compiled package with legacy code](https://github.com/spectre-team/matlab-legacy/releases/tag/legacy-v5.0.0)
-
-Installation process may be clearer with insight into Docker images used for
-application deployment:
-
-- [`python_mcr` image](https://github.com/spectre-team/python_mcr) - installs
-MCR r2019a onto Python 3.7 image
-- [`python_msi` image](https://github.com/spectre-team/python_msi) - installs
-compiled legacy code onto MCR image
-- [`divik` image](https://github.com/spectre-team/spectre-divik/blob/master/dockerfile) -
-installs DiviK software onto legacy code image
+- compiler capable of compiling the native C code
 
 Having prerequisites installed, one can install latest base version of the
 package:
@@ -75,7 +59,7 @@ pip install divik
 or any stable tagged version, e.g.:
 
 ```bash
-pip install divik==2.3.6
+pip install divik==2.3.8
 ```
 
 # References

dev_setup.py

@@ -0,0 +1,22 @@
+"""This just builds the native extension for testing purposes"""
+
+from glob import glob
+from setuptools import setup, Extension
+import numpy
+
+setup(
+    packages=[],
+    # @gmrukwa: https://packaging.python.org/discussions/install-requires-vs-requirements/
+    install_requires=[
+        'numpy>=0.12.1',
+    ],
+    python_requires='>=3.5,<=3.7',
+    ext_modules=[
+        Extension('gamred_native',
+                  sources=glob('gamred_native/*.c'),
+                  include_dirs=['gamred_native', numpy.get_include()],
+                  define_macros=[
+                      ('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION'),
+                  ]),
+    ],
+)

divik/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '2.3.6'
+__version__ = '2.3.8'
 
 from ._seeding import seeded
 from ._utils import DivikResult

divik/_matlab_legacy.py

@@ -1,102 +1,34 @@
 """Access to MATLAB legacy functionalities."""
-
-from contextlib import contextmanager
-import logging
-import os
-import platform
-import warnings
-
+import gamred_native as gn
 import numpy as np
 
 
-# noinspection SpellCheckingInspection
-_MATLAB_SEARCH_PATHS = \
-    "/usr/local/MATLAB/MATLAB_Runtime/v96/runtime/glnxa64:" + \
-    "/usr/local/MATLAB/MATLAB_Runtime/v96/bin/glnxa64:" + \
-    "/usr/local/MATLAB/MATLAB_Runtime/v96/sys/os/glnxa64:" + \
-    "/usr/local/MATLAB/MATLAB_Runtime/v96/extern/bin/glnxa64:"
-
-
-_local_system = platform.system()
-
-if _local_system == 'Windows':
-    # Must be here. Doesn't work as contextmanager.
-    # If you think different increase counter of wasted hours: 4
-    os.environ['PATH'] = os.environ['PATH'].lower()
-
-
-_logger = logging.getLogger(__name__)
-
-
-@contextmanager
-def _matlab_paths():
-    if _local_system == 'Linux':
-        _logger.log(logging.INFO, 'Modifying LD_LIBRARY_PATH.')
-        old_env = os.environ.get('LD_LIBRARY_PATH', '')
-        os.environ['LD_LIBRARY_PATH'] = _MATLAB_SEARCH_PATHS + old_env
-        _logger.log(logging.INFO, os.environ['LD_LIBRARY_PATH'])
-    elif _local_system == 'Darwin':
-        raise NotImplementedError('OSX hosts are not supported.')
-    try:
-        yield
-    finally:
-        if _local_system == 'Linux':
-            _logger.log(logging.INFO, 'Restoring LD_LIBRARY_PATH.')
-            # noinspection PyUnboundLocalVariable
-            os.environ['LD_LIBRARY_PATH'] = old_env
-            _logger.log(logging.INFO, os.environ['LD_LIBRARY_PATH'])
-
-
-_engine = None
-
-
-def _ensure_engine():
-    global _engine
-    if _engine is None:
-        with _matlab_paths():
-            import MatlabAlgorithms.MsiAlgorithms
-            # noinspection PyPackageRequirements
-            import matlab
-        _engine = MatlabAlgorithms.MsiAlgorithms.initialize()
-    return _engine
-
-
-class MatlabError(Exception):
-    """Thrown when legacy computations failed."""
-
-    pass
+def find_thresholds(values: np.ndarray, max_components: int = 10) \
+        -> np.ndarray:
+    """Find candidate thresholds for decomposition of values by GMM.
 
+    Parameters
+    ----------
+    values:
+        vector of values to decompose
 
-def find_thresholds(values: np.ndarray, max_components: int = 10,
-                    throw_on_engine_error: bool = True) -> np.ndarray:
-    """Find candidate thresholds for decomposition of values by GMM.
+    max_components:
+        maximal number of components to decompose into
 
-    @param values: vector of values to decompose
-    @param max_components: maximal number of components to decompose into
-    @param throw_on_engine_error: if true, an exception will be raised if
-    legacy code fails. Returns empty thresholds array otherwise.
-    @return: array of candidate thresholds from crossings between GMM
-    components
+    Returns
+    -------
+    array of candidate thresholds from crossings between GMM components
     """
-    with _matlab_paths():
-        engine = _ensure_engine()
-        # noinspection PyUnresolvedReferences
-        import MatlabAlgorithms.MsiAlgorithms
-        # noinspection PyPackageRequirements
-        import matlab
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore")
-            values = matlab.double([[element] for element in values.ravel()])
-        try:
-            thresholds = engine.fetch_thresholds(values,
-                                                 'MaxComponents',
-                                                 float(max_components),
-                                                 'DisableWarnings',
-                                                 True,
-                                                 nargout=1)
-        except Exception as ex:
-            if throw_on_engine_error:
-                raise MatlabError() from ex
-            else:
-                return np.array([])
-        return np.array(thresholds).ravel()
+    if values.size <= max_components:
+        max_components = values.size
+    if values.size == 0:
+        return np.array([])
+    if max_components <= 0:
+        raise ValueError("max_components must be positive")
+    if values.ndim != 1:
+        raise ValueError("values must be 1D array")
+    values = np.ascontiguousarray(values)
+    offset = np.min(values)
+    if np.min(values) == np.max(values):
+        return np.array([])
+    return gn.find_thresholds(values - offset, max_components) + offset

divik/feature_selection/_gmm_selector.py

@@ -114,10 +114,8 @@ def fit(self, X, y=None):
         self
         """
         self.vals_ = self._to_characteristics(X)
-        thrs = ml.find_thresholds(  # the translation is due to MATLAB's problem
-            self.vals_ - self.vals_.min(),
-            max_components=self.max_components,
-            throw_on_engine_error=False) + self.vals_.min()
+        thrs = ml.find_thresholds(
+            self.vals_, max_components=self.max_components)
         n_candidates = len(thrs) if self.n_candidates is None \
             else self.n_candidates
         desired_thrs = thrs[:n_candidates]

docker/deploy.dockerfile

@@ -1,4 +1,8 @@
-FROM spectreteam/python_msi:v5.1.0.2019a.py37
+FROM python:3.7-slim
+
+RUN apt-get update &&\
+    apt-get install -y gcc &&\
+    rm -rf /var/lib/apt/lists/*
 
 ENV PYTHONUNBUFFERED TRUE
 

docker/development.dockerfile

@@ -1,4 +1,8 @@
-FROM spectreteam/python_msi:v5.1.0.2019a.py37
+FROM python:3.7-slim
+
+RUN apt-get update &&\
+    apt-get install -y gcc &&\
+    rm -rf /var/lib/apt/lists/*
 
 ENV PYTHONUNBUFFERED TRUE
 
@@ -9,3 +13,13 @@ RUN pip install --no-cache-dir -r /requirements.txt &&\
 
 RUN mkdir -p /root/.config/matplotlib &&\
   echo "backend : Agg" > /root/.config/matplotlib/matplotlibrc
+
+WORKDIR /app
+
+COPY gamred_native /app/gamred_native
+
+COPY dev_setup.py /app/dev_setup.py
+
+RUN python dev_setup.py install &&\
+    rm -rf /app/gamred_native &&\
+    rm /app/dev_setup.py