diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0064445..596cf67 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,8 +57,6 @@ jobs: - if: startsWith(matrix.python, '2') run: pytest - run: codecov - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} matlab: if: github.event_name != 'pull_request' || github.head_ref != 'devel' runs-on: [self-hosted, python, cuda, matlab] @@ -72,8 +70,6 @@ jobs: - run: pip install -U .[dev,nii,cuda,web,mbeautify] - run: pytest --durations-min=1 - run: codecov - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Post Run setup-python run: setup-python -p3.7 -Dr if: ${{ always() }} diff --git a/miutil/cuinfo.py b/miutil/cuinfo.py index 970816b..e8dc80f 100755 --- a/miutil/cuinfo.py +++ b/miutil/cuinfo.py @@ -18,9 +18,17 @@ def nvmlDeviceGetCudaComputeCapability(handle): major = pynvml.c_int() minor = pynvml.c_int() - fn = pynvml.get_func_pointer("nvmlDeviceGetCudaComputeCapability") + try: # pynvml>=11 + get_fn = pynvml.nvml._nvmlGetFunctionPointer + except AttributeError: + get_fn = pynvml.get_func_pointer + fn = get_fn("nvmlDeviceGetCudaComputeCapability") ret = fn(handle, pynvml.byref(major), pynvml.byref(minor)) - pynvml.check_return(ret) + try: # pynvml>=11 + check_ret = pynvml.nvml._nvmlCheckReturn + except AttributeError: + check_ret = pynvml.check_return + check_ret(ret) return [major.value, minor.value] diff --git a/miutil/fdio.py b/miutil/fdio.py index ed6fef9..d52326b 100644 --- a/miutil/fdio.py +++ b/miutil/fdio.py @@ -66,5 +66,10 @@ def extractall(fzip, dest, desc="Extracting"): if not getattr(i, "file_size", 0): # directory zipf.extract(i, fspath(dest)) else: - with zipf.open(i) as fi, open(fspath(dest / i.filename), "wb") as fo: + (dest / i.filename).parent.mkdir(parents=True, exist_ok=True) + with zipf.open(i) as fi, (dest / i.filename).open(mode="wb") as fo: copyfileobj(CallbackIOWrapper(pbar.update, fi), fo) + mode = (i.external_attr >> 16) & 0o777 + if mode: + (dest / i.filename).chmod(mode) + log.debug(oct((i.external_attr >> 16) & 0o777)) diff --git a/miutil/mlab/__init__.py b/miutil/mlab/__init__.py index e273a6b..b697c53 100644 --- a/miutil/mlab/__init__.py +++ b/miutil/mlab/__init__.py @@ -1,8 +1,10 @@ import logging +import os import re import sys from ast import literal_eval from os import getenv, path +from platform import system from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent @@ -16,7 +18,7 @@ FileNotFoundError = OSError -from ..fdio import tmpdir +from ..fdio import Path, extractall, fspath, tmpdir __all__ = ["get_engine"] IS_WIN = any(sys.platform.startswith(i) for i in ["win32", "cygwin"]) @@ -24,6 +26,28 @@ if IS_WIN: MATLAB_RUN += ["-wait", "-log"] log = logging.getLogger(__name__) +_MCR_URL = { + 99: ( + "https://ssd.mathworks.com/supportfiles/downloads/R2020b/Release/4" + "/deployment_files/installer/complete/" + ), + 713: "https://www.fil.ion.ucl.ac.uk/spm/download/restricted/utopia/MCR/", +} +MCR_ARCH = {"Windows": "win64", "Linux": "glnxa64", "Darwin": "maci64"}[system()] +MCR_URL = { + "Windows": { + 99: _MCR_URL[99] + "win64/MATLAB_Runtime_R2020b_Update_4_win64.zip", + 713: _MCR_URL[713] + "win64/MCRInstaller.exe", + }, + "Linux": { + 99: _MCR_URL[99] + "glnxa64/MATLAB_Runtime_R2020b_Update_4_glnxa64.zip", + 713: _MCR_URL[713] + "glnxa64/MCRInstaller.bin", + }, + "Darwin": { + 99: _MCR_URL[99] + "maci64/MATLAB_Runtime_R2020b_Update_4_maci64.dmg.zip", + 713: _MCR_URL[713] + "maci64/MCRInstaller.dmg", + }, +}[system()] class VersionError(ValueError): @@ -34,6 +58,13 @@ def check_output_u8(*args, **kwargs): return check_output(*args, **kwargs).decode("utf-8").strip() +def env_prefix(key, dir): + try: + os.environ[key] = "%s%s%s" % (os.environ[key], os.pathsep, fspath(dir)) + except KeyError: + os.environ[key] = str(fspath(dir)) + + @lru_cache() def get_engine(name=None): try: @@ -69,6 +100,8 @@ def get_engine(name=None): (e.g. /tmp/builddir). - If installation fails due to write permissions, try appending `--user` to the above command. + + Alternatively, use `get_runtime()` instead of `get_engine()`. """ ).format( matlabroot=matlabroot(default="matlabroot"), exe=sys.executable @@ -136,3 +169,88 @@ def _install_engine(): except CalledProcessError: log.warning("Normal install failed. Attempting `--user` install.") return check_output_u8(cmd + ["--user"], cwd=src) + + +@lru_cache() +def get_runtime(cache="~/.mcr", version=99): + cache = Path(cache).expanduser() + mcr_root = cache + i = mcr_root / ("v%d" % version) + if i.is_dir(): + mcr_root = i + else: + from miutil.web import urlopen_cached + + log.info("Downloading to %s", cache) + with tmpdir() as td: + with urlopen_cached(MCR_URL[version], cache) as fd: + if MCR_URL[version].endswith(".zip"): + extractall(fd, td) + log.info("Installing ... (may take a few min)") + if version == 99: + check_output_u8( + [ + fspath( + Path(td) / ("setup" if system() == "Windows" else "install") + ), + "-mode", + "silent", + "-agreeToLicense", + "yes", + "-destinationFolder", + fspath(mcr_root), + ] + ) + elif version == 713: + install = cache / MCR_URL[version].rsplit("/", 1)[-1] + if system() == "Linux": + install.chmod(0o755) + check_output_u8( + [ + fspath(install), + "-P", + 'bean421.installLocation="%s"' % fspath(cache), + "-silent", + ] + ) + else: + raise NotImplementedError( + dedent( + """\ + Don't yet know how to handle + {} + for {!r}. + """ + ).format(fspath(install), system()) + ) + else: + raise IndexError(version) + mcr_root /= "v%d" % version + log.info("Installed") + + # bin + if (mcr_root / "bin" / MCR_ARCH).is_dir(): + env_prefix("PATH", mcr_root / "bin" / MCR_ARCH) + else: + log.warning("Cannot find MCR bin") + + # libs + env_var = { + "Linux": "LD_LIBRARY_PATH", + "Windows": "PATH", + "Darwin": "DYLD_LIBRARY_PATH", + }[system()] + if (mcr_root / "runtime" / MCR_ARCH).is_dir(): + env_prefix(env_var, mcr_root / "runtime" / MCR_ARCH) + else: + log.warning("Cannot find MCR libs") + + # python module + pydist = mcr_root / "extern" / "engines" / "python" / "dist" + if pydist.is_dir(): + if fspath(pydist) not in sys.path: + sys.path.insert(1, fspath(pydist)) + else: + log.warning("Cannot find MCR Python dist") + + return mcr_root diff --git a/miutil/mlab/beautify.py b/miutil/mlab/beautify.py index 4dcb31f..e8d492a 100755 --- a/miutil/mlab/beautify.py +++ b/miutil/mlab/beautify.py @@ -17,7 +17,7 @@ from . import get_engine, lru_cache log = logging.getLogger(__name__) -MBEAUTIFIER_REV = "9c3c82387ec3c0fb29e707ebd060e8e2ca9ea6f2" +MBEAUTIFIER_REV = "1a57849e44662f56271dc0eefa746855698a719a" @lru_cache() diff --git a/setup.cfg b/setup.cfg index caee3d1..6a61a62 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,8 +7,8 @@ license=Apache 2.0 license_file=LICENCE.md url=https://github.com/AMYPAD/miutil project_urls= - Changelog = https://github.com/AMYPAD/miutil/releases - Documentation = https://github.com/AMYPAD/miutil/#miutil + Changelog=https://github.com/AMYPAD/miutil/releases + Documentation=https://github.com/AMYPAD/miutil/#miutil maintainer=Casper da Costa-Luis maintainer_email=casper.dcl@physics.org keywords=fMRI, PET, SPECT, EEG, MEG @@ -43,7 +43,7 @@ classifiers= setup_requires=setuptools>=42; wheel; setuptools_scm[toml]>=3.4 install_requires= pathlib2; python_version <= "2.7" - tqdm + tqdm>=4.40.0 packages=find: python_requires=>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.* [options.extras_require] @@ -56,18 +56,19 @@ dev= pytest-timeout pytest-xdist codecov -nii = nibabel; numpy -plot = matplotlib; numpy; scipy -cuda = argopt; pynvml -web = requests -mbeautify = argopt; %(web)s +nii=nibabel; numpy +plot=matplotlib; numpy; scipy +cuda=argopt; pynvml +web=requests +mbeautify=argopt; tqdm>=4.42.0; %(web)s [options.entry_points] -console_scripts = - cuinfo = miutil.cuinfo:main - mbeautify = miutil.mlab.beautify:main - +console_scripts= + cuinfo=miutil.cuinfo:main + mbeautify=miutil.mlab.beautify:main +[options.packages.find] +exclude=tests [bdist_wheel] -universal = 1 +universal=1 [flake8] max_line_length=88 diff --git a/tests/test_mlab.py b/tests/test_mlab.py index 04d1df3..449d6d5 100644 --- a/tests/test_mlab.py +++ b/tests/test_mlab.py @@ -1,4 +1,4 @@ -from pytest import fixture, importorskip, skip +from pytest import fixture, importorskip, mark, skip @fixture @@ -32,6 +32,19 @@ def test_engine(eng): assert eng == eng2 +@mark.timeout(3600) +def test_runtime(): + importorskip("miutil.web") + from miutil.mlab import get_runtime + + mcr_root = get_runtime() + for i in ("bin", "extern", "mcr", "runtime", "sys", "toolbox"): + assert (mcr_root / i).is_dir() + + mcr_root2 = get_runtime() + assert mcr_root == mcr_root2 + + def test_beautify(eng): beautify = importorskip("miutil.mlab.beautify")