From 81292f31bedd89875853dd801765efb7f052dc2e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 13 Feb 2021 13:38:20 +0000 Subject: [PATCH 01/11] fdio.extractall: fix permissions --- miutil/fdio.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/miutil/fdio.py b/miutil/fdio.py index ed6fef9..acddd12 100644 --- a/miutil/fdio.py +++ b/miutil/fdio.py @@ -66,5 +66,7 @@ def extractall(fzip, dest, desc="Extracting"): if not getattr(i, "file_size", 0): # directory zipf.extract(i, fspath(dest)) else: - with zipf.open(i) as fi, open(fspath(dest / i.filename), "wb") as fo: + (dest / i.filename).parent.mkdir(parents=True, exist_ok=True) + with zipf.open(i) as fi, (dest / i.filename).open(mode="wb") as fo: copyfileobj(CallbackIOWrapper(pbar.update, fi), fo) + (dest / i.filename).chmod((i.external_attr >> 16) & 0o777) From d25837cecec8ca0bf27ec2d98543692bf1d82787 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 13 Feb 2021 13:48:13 +0000 Subject: [PATCH 02/11] mlab: add `get_runtime()` --- miutil/mlab/__init__.py | 73 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/miutil/mlab/__init__.py b/miutil/mlab/__init__.py index e273a6b..74341c6 100644 --- a/miutil/mlab/__init__.py +++ b/miutil/mlab/__init__.py @@ -1,8 +1,10 @@ import logging +import os import re import sys from ast import literal_eval from os import getenv, path +from platform import system from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent @@ -16,7 +18,7 @@ FileNotFoundError = OSError -from ..fdio import tmpdir +from ..fdio import Path, extractall, fspath, tmpdir __all__ = ["get_engine"] IS_WIN = any(sys.platform.startswith(i) for i in ["win32", "cygwin"]) @@ -24,6 +26,16 @@ if IS_WIN: MATLAB_RUN += ["-wait", "-log"] log = logging.getLogger(__name__) +_MCR_URL = ( + "https://ssd.mathworks.com/supportfiles/downloads/R2020b/Release/4" + "/deployment_files/installer/complete/" +) +MCR_ARCH = {"Windows": "win64", "Linux": "glnxa64", "Darwin": "maci64"}[system()] +MCR_URL = { + "Windows": _MCR_URL + "win64/MATLAB_Runtime_R2020b_Update_4_win64.zip", + "Linux": _MCR_URL + "glnxa64/MATLAB_Runtime_R2020b_Update_4_glnxa64.zip", + "Darwin": _MCR_URL + "maci64/MATLAB_Runtime_R2020b_Update_4_maci64.dmg.zip", +}[system()] class VersionError(ValueError): @@ -34,6 +46,10 @@ def check_output_u8(*args, **kwargs): return check_output(*args, **kwargs).decode("utf-8").strip() +def env_prefix(key, dir): + os.environ[key] = "%s%s%s" % (os.environ[key], os.pathsep, fspath(dir)) + + @lru_cache() def get_engine(name=None): try: @@ -69,6 +85,8 @@ def get_engine(name=None): (e.g. /tmp/builddir). - If installation fails due to write permissions, try appending `--user` to the above command. + + Alternatively, use `get_runtime()` instead of `get_engine()`. """ ).format( matlabroot=matlabroot(default="matlabroot"), exe=sys.executable @@ -136,3 +154,56 @@ def _install_engine(): except CalledProcessError: log.warning("Normal install failed. Attempting `--user` install.") return check_output_u8(cmd + ["--user"], cwd=src) + + +@lru_cache() +def get_runtime(cache="~/.mcr"): + cache = Path(cache).expanduser() + mcr_root = cache + try: + mcr_root = max(i for i in mcr_root.glob("v*") if i.is_dir()) + except ValueError: + from miutil.web import urlopen_cached + + log.info("Downloading to %s", cache) + with tmpdir() as td: + with urlopen_cached(MCR_URL, cache) as fd: + extractall(fd, td) + inst_opts = [ + "-mode", + "silent", + "-agreeToLicense", + "yes", + "-destinationFolder", + fspath(mcr_root), + ] + check_output_u8( + [fspath(Path(td) / ("setup" if system() == "Windows" else "install"))] + + inst_opts + ) + # bin + if not (mcr_root / "bin" / MCR_ARCH).is_dir(): + log.warning("Cannot find MCR bin") + env_prefix("PATH", mcr_root / "bin" / MCR_ARCH) + + # libs + env_var = { + "Linux": "LD_LIBRARY_PATH", + "Windows": "PATH", + "Darwin": "DYLD_LIBRARY_PATH", + }[system()] + if not (mcr_root / "runtime" / MCR_ARCH).is_dir(): + log.warning("Cannot find MCR libs") + env_prefix(env_var, mcr_root / "runtime" / MCR_ARCH) + + # python module + pydist = mcr_root / "extern" / "engines" / "python" / "dist" + if not pydist.is_dir(): + log.warning("Cannot find MCR Python dist") + if fspath(pydist) not in sys.path: + sys.path.insert(1, fspath(pydist)) + + log.info("Installed") + import matlab + + return matlab From d2f9eaf7e49b7f38c6663cc4e6bc2bf4585aa80f Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 13 Feb 2021 13:56:35 +0000 Subject: [PATCH 03/11] fdio: fix `extractall` mode --- miutil/fdio.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/miutil/fdio.py b/miutil/fdio.py index acddd12..d52326b 100644 --- a/miutil/fdio.py +++ b/miutil/fdio.py @@ -69,4 +69,7 @@ def extractall(fzip, dest, desc="Extracting"): (dest / i.filename).parent.mkdir(parents=True, exist_ok=True) with zipf.open(i) as fi, (dest / i.filename).open(mode="wb") as fo: copyfileobj(CallbackIOWrapper(pbar.update, fi), fo) - (dest / i.filename).chmod((i.external_attr >> 16) & 0o777) + mode = (i.external_attr >> 16) & 0o777 + if mode: + (dest / i.filename).chmod(mode) + log.debug(oct((i.external_attr >> 16) & 0o777)) From c5975b687ad5875e2c883d2c982538a7dae41a0f Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 13 Feb 2021 14:15:34 +0000 Subject: [PATCH 04/11] tests: basic mlab runtime --- tests/test_mlab.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_mlab.py b/tests/test_mlab.py index 04d1df3..0954282 100644 --- a/tests/test_mlab.py +++ b/tests/test_mlab.py @@ -1,4 +1,4 @@ -from pytest import fixture, importorskip, skip +from pytest import fixture, importorskip, mark, skip @fixture @@ -32,6 +32,16 @@ def test_engine(eng): assert eng == eng2 +@mark.timeout(3600) +def test_runtime(): + importorskip("miutil.web") + from miutil.mlab import get_runtime + + matlab = get_runtime() + matlab2 = get_runtime() + assert matlab == matlab2 + + def test_beautify(eng): beautify = importorskip("miutil.mlab.beautify") From fdd00470f002af85b52bd199e491d4f515eae03d Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 13 Feb 2021 14:24:03 +0000 Subject: [PATCH 05/11] tests: matlab runtime root --- miutil/mlab/__init__.py | 3 +-- tests/test_mlab.py | 9 ++++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/miutil/mlab/__init__.py b/miutil/mlab/__init__.py index 74341c6..1adf2c0 100644 --- a/miutil/mlab/__init__.py +++ b/miutil/mlab/__init__.py @@ -204,6 +204,5 @@ def get_runtime(cache="~/.mcr"): sys.path.insert(1, fspath(pydist)) log.info("Installed") - import matlab - return matlab + return mcr_root diff --git a/tests/test_mlab.py b/tests/test_mlab.py index 0954282..449d6d5 100644 --- a/tests/test_mlab.py +++ b/tests/test_mlab.py @@ -37,9 +37,12 @@ def test_runtime(): importorskip("miutil.web") from miutil.mlab import get_runtime - matlab = get_runtime() - matlab2 = get_runtime() - assert matlab == matlab2 + mcr_root = get_runtime() + for i in ("bin", "extern", "mcr", "runtime", "sys", "toolbox"): + assert (mcr_root / i).is_dir() + + mcr_root2 = get_runtime() + assert mcr_root == mcr_root2 def test_beautify(eng): From 830bad970e5ae35da22401f73395ef3501e62e74 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 13 Feb 2021 17:43:44 +0000 Subject: [PATCH 06/11] mlab.get_runtime: add v713 Linux install --- miutil/mlab/__init__.py | 113 ++++++++++++++++++++++++++++------------ 1 file changed, 79 insertions(+), 34 deletions(-) diff --git a/miutil/mlab/__init__.py b/miutil/mlab/__init__.py index 1adf2c0..0e25733 100644 --- a/miutil/mlab/__init__.py +++ b/miutil/mlab/__init__.py @@ -26,15 +26,27 @@ if IS_WIN: MATLAB_RUN += ["-wait", "-log"] log = logging.getLogger(__name__) -_MCR_URL = ( - "https://ssd.mathworks.com/supportfiles/downloads/R2020b/Release/4" - "/deployment_files/installer/complete/" -) +_MCR_URL = { + 99: ( + "https://ssd.mathworks.com/supportfiles/downloads/R2020b/Release/4" + "/deployment_files/installer/complete/" + ), + 713: "https://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/MCR/", +} MCR_ARCH = {"Windows": "win64", "Linux": "glnxa64", "Darwin": "maci64"}[system()] MCR_URL = { - "Windows": _MCR_URL + "win64/MATLAB_Runtime_R2020b_Update_4_win64.zip", - "Linux": _MCR_URL + "glnxa64/MATLAB_Runtime_R2020b_Update_4_glnxa64.zip", - "Darwin": _MCR_URL + "maci64/MATLAB_Runtime_R2020b_Update_4_maci64.dmg.zip", + "Windows": { + 99: _MCR_URL[99] + "win64/MATLAB_Runtime_R2020b_Update_4_win64.zip", + 713: _MCR_URL[713] + "win64/MCRInstaller.exe", + }, + "Linux": { + 99: _MCR_URL[99] + "glnxa64/MATLAB_Runtime_R2020b_Update_4_glnxa64.zip", + 713: _MCR_URL[713] + "glnxa64/MCRInstaller.bin", + }, + "Darwin": { + 99: _MCR_URL[99] + "maci64/MATLAB_Runtime_R2020b_Update_4_maci64.dmg.zip", + 713: _MCR_URL[713] + "maci64/MCRInstaller.dmg", + }, }[system()] @@ -157,34 +169,67 @@ def _install_engine(): @lru_cache() -def get_runtime(cache="~/.mcr"): +def get_runtime(cache="~/.mcr", version=99): cache = Path(cache).expanduser() mcr_root = cache - try: - mcr_root = max(i for i in mcr_root.glob("v*") if i.is_dir()) - except ValueError: + i = mcr_root / ("v%d" % version) + if i.is_dir(): + mcr_root = i + else: from miutil.web import urlopen_cached log.info("Downloading to %s", cache) with tmpdir() as td: - with urlopen_cached(MCR_URL, cache) as fd: - extractall(fd, td) - inst_opts = [ - "-mode", - "silent", - "-agreeToLicense", - "yes", - "-destinationFolder", - fspath(mcr_root), - ] - check_output_u8( - [fspath(Path(td) / ("setup" if system() == "Windows" else "install"))] - + inst_opts - ) + with urlopen_cached(MCR_URL[version], cache) as fd: + if MCR_URL[version].endswith(".zip"): + extractall(fd, td) + log.info("Installing ... (may take a few min)") + if version == 99: + check_output_u8( + [ + fspath( + Path(td) / ("setup" if system() == "Windows" else "install") + ), + "-mode", + "silent", + "-agreeToLicense", + "yes", + "-destinationFolder", + fspath(mcr_root), + ] + ) + elif version == 713: + install = cache / MCR_URL[version].rsplit("/", 1)[-1] + if system() == "Linux": + install.chmod(0o755) + check_output_u8( + [ + fspath(install), + "-P", + 'bean421.installLocation="%s"' % fspath(cache), + "-silent", + ] + ) + else: + raise NotImplementedError( + dedent( + """\ + Don't yet know how to handle + {} + for {!r}. + """ + ).format(fspath(install), system()) + ) + else: + raise IndexError(version) + mcr_root /= "v%d" % version + log.info("Installed") + # bin - if not (mcr_root / "bin" / MCR_ARCH).is_dir(): + if (mcr_root / "bin" / MCR_ARCH).is_dir(): + env_prefix("PATH", mcr_root / "bin" / MCR_ARCH) + else: log.warning("Cannot find MCR bin") - env_prefix("PATH", mcr_root / "bin" / MCR_ARCH) # libs env_var = { @@ -192,17 +237,17 @@ def get_runtime(cache="~/.mcr"): "Windows": "PATH", "Darwin": "DYLD_LIBRARY_PATH", }[system()] - if not (mcr_root / "runtime" / MCR_ARCH).is_dir(): + if (mcr_root / "runtime" / MCR_ARCH).is_dir(): + env_prefix(env_var, mcr_root / "runtime" / MCR_ARCH) + else: log.warning("Cannot find MCR libs") - env_prefix(env_var, mcr_root / "runtime" / MCR_ARCH) # python module pydist = mcr_root / "extern" / "engines" / "python" / "dist" - if not pydist.is_dir(): + if pydist.is_dir(): + if fspath(pydist) not in sys.path: + sys.path.insert(1, fspath(pydist)) + else: log.warning("Cannot find MCR Python dist") - if fspath(pydist) not in sys.path: - sys.path.insert(1, fspath(pydist)) - - log.info("Installed") return mcr_root From 93f1de63f73832998e18513c4f2aac359be0304b Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 13 Feb 2021 18:35:21 +0000 Subject: [PATCH 07/11] mlab fix env KeyError --- miutil/mlab/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/miutil/mlab/__init__.py b/miutil/mlab/__init__.py index 0e25733..b697c53 100644 --- a/miutil/mlab/__init__.py +++ b/miutil/mlab/__init__.py @@ -59,7 +59,10 @@ def check_output_u8(*args, **kwargs): def env_prefix(key, dir): - os.environ[key] = "%s%s%s" % (os.environ[key], os.pathsep, fspath(dir)) + try: + os.environ[key] = "%s%s%s" % (os.environ[key], os.pathsep, fspath(dir)) + except KeyError: + os.environ[key] = str(fspath(dir)) @lru_cache() From 80940b8bad13aa148e6c73e35f915f1258a79875 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 20 Feb 2021 22:39:59 +0000 Subject: [PATCH 08/11] tests: remove unneeded token --- .github/workflows/test.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0064445..596cf67 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,8 +57,6 @@ jobs: - if: startsWith(matrix.python, '2') run: pytest - run: codecov - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} matlab: if: github.event_name != 'pull_request' || github.head_ref != 'devel' runs-on: [self-hosted, python, cuda, matlab] @@ -72,8 +70,6 @@ jobs: - run: pip install -U .[dev,nii,cuda,web,mbeautify] - run: pytest --durations-min=1 - run: codecov - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Post Run setup-python run: setup-python -p3.7 -Dr if: ${{ always() }} From 4d0cb767bf221f95a99c6d73d7a940e5bc3fe4fc Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 29 Apr 2021 13:49:01 +0100 Subject: [PATCH 09/11] bump MBeautifier version Fixes https://github.com/davidvarga/MBeautifier/issues/99 --- miutil/mlab/beautify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/miutil/mlab/beautify.py b/miutil/mlab/beautify.py index 4dcb31f..e8d492a 100755 --- a/miutil/mlab/beautify.py +++ b/miutil/mlab/beautify.py @@ -17,7 +17,7 @@ from . import get_engine, lru_cache log = logging.getLogger(__name__) -MBEAUTIFIER_REV = "9c3c82387ec3c0fb29e707ebd060e8e2ca9ea6f2" +MBEAUTIFIER_REV = "1a57849e44662f56271dc0eefa746855698a719a" @lru_cache() From 869a320cae42857305d4f86ce0330c9127a07997 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 25 May 2021 11:15:22 +0100 Subject: [PATCH 10/11] fix deps, no dist tests - fixes #21 --- setup.cfg | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/setup.cfg b/setup.cfg index caee3d1..6a61a62 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,8 +7,8 @@ license=Apache 2.0 license_file=LICENCE.md url=https://github.com/AMYPAD/miutil project_urls= - Changelog = https://github.com/AMYPAD/miutil/releases - Documentation = https://github.com/AMYPAD/miutil/#miutil + Changelog=https://github.com/AMYPAD/miutil/releases + Documentation=https://github.com/AMYPAD/miutil/#miutil maintainer=Casper da Costa-Luis maintainer_email=casper.dcl@physics.org keywords=fMRI, PET, SPECT, EEG, MEG @@ -43,7 +43,7 @@ classifiers= setup_requires=setuptools>=42; wheel; setuptools_scm[toml]>=3.4 install_requires= pathlib2; python_version <= "2.7" - tqdm + tqdm>=4.40.0 packages=find: python_requires=>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.* [options.extras_require] @@ -56,18 +56,19 @@ dev= pytest-timeout pytest-xdist codecov -nii = nibabel; numpy -plot = matplotlib; numpy; scipy -cuda = argopt; pynvml -web = requests -mbeautify = argopt; %(web)s +nii=nibabel; numpy +plot=matplotlib; numpy; scipy +cuda=argopt; pynvml +web=requests +mbeautify=argopt; tqdm>=4.42.0; %(web)s [options.entry_points] -console_scripts = - cuinfo = miutil.cuinfo:main - mbeautify = miutil.mlab.beautify:main - +console_scripts= + cuinfo=miutil.cuinfo:main + mbeautify=miutil.mlab.beautify:main +[options.packages.find] +exclude=tests [bdist_wheel] -universal = 1 +universal=1 [flake8] max_line_length=88 From ca5db1e4f6c64c6a3562ebff23700fb6daa703bd Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 13 Jun 2021 18:28:10 +0100 Subject: [PATCH 11/11] fix pynvml==11 AttributeError - fixes #22 --- miutil/cuinfo.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/miutil/cuinfo.py b/miutil/cuinfo.py index 970816b..e8dc80f 100755 --- a/miutil/cuinfo.py +++ b/miutil/cuinfo.py @@ -18,9 +18,17 @@ def nvmlDeviceGetCudaComputeCapability(handle): major = pynvml.c_int() minor = pynvml.c_int() - fn = pynvml.get_func_pointer("nvmlDeviceGetCudaComputeCapability") + try: # pynvml>=11 + get_fn = pynvml.nvml._nvmlGetFunctionPointer + except AttributeError: + get_fn = pynvml.get_func_pointer + fn = get_fn("nvmlDeviceGetCudaComputeCapability") ret = fn(handle, pynvml.byref(major), pynvml.byref(minor)) - pynvml.check_return(ret) + try: # pynvml>=11 + check_ret = pynvml.nvml._nvmlCheckReturn + except AttributeError: + check_ret = pynvml.check_return + check_ret(ret) return [major.value, minor.value]