From c85dd41e8616052f24db7837c3be8d67fc079ef0 Mon Sep 17 00:00:00 2001 From: AbdealiJK Date: Sun, 26 Jun 2016 08:12:10 +0530 Subject: [PATCH] setup: Revamp setup to handle deps better Have a class for every dependency which checks for the required packages correctly. --- .travis.yml | 8 +- MANIFEST.in | 1 - requirements.txt | 21 -- setup.py | 148 ++++++++----- setupdeps.py | 554 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 650 insertions(+), 82 deletions(-) delete mode 100644 requirements.txt create mode 100644 setupdeps.py diff --git a/.travis.yml b/.travis.yml index 73d8d7a..a407f1f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,14 +53,14 @@ install: # Install dlib from git (with pip) as there was an issue with libjpeg # Which was solved and not yet released. - pip install git+https://github.com/davisking/dlib.git@02f6da285149a61bc58728d9c5e77932151118b5#egg=dlib ; - - pip install -r test-requirements.txt -r requirements.txt ; + - pip install -r test-requirements.txt ; + - pip install . + - pip uninstall -y file-metadata script: - - flake8 setup.py file_metadata tests + - flake8 setup.py setupdeps.py file_metadata tests - python -m pytest --cov ; - python setup.py sdist bdist bdist_wheel - - pip install . - - pip uninstall -y file-metadata after_script: - bash <(curl -s https://codecov.io/bash) diff --git a/MANIFEST.in b/MANIFEST.in index e3b30a7..7a8b3c5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ include README.* include LICENSE include test-requirements.txt -include requirements.txt include setup.cfg diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index fe71682..0000000 --- a/requirements.txt +++ /dev/null @@ -1,21 +0,0 @@ -setuptools - -# For backporting -pathlib2 -backports.lzma - -# Not directly used by us, but required by dependencies -matplotlib>=1.3.1 - -appdirs>=1.1.0 -pycolorname>=0.1.0 - -python-magic -six>=1.8.0 -pillow>=2.5.0 -numpy>=1.7.2 -scipy>=0.9.0 -dlib -scikit-image>=0.12 -wand -zbar diff --git a/setup.py b/setup.py index 7b0b6b2..04401ec 100644 --- a/setup.py +++ b/setup.py @@ -5,72 +5,107 @@ print_function) import os -import platform -import subprocess import sys +from distutils import log +from distutils.errors import DistutilsSetupError -from setuptools import find_packages, setup +import setupdeps +log.set_verbosity(log.INFO) -with open(os.devnull, 'w') as nul: - # Check if exiftool or perl is installed. - try: - out = subprocess.check_call(['perl', '-v'], stdout=nul, stderr=nul) - except (OSError, subprocess.CalledProcessError): - try: - out = subprocess.check_call(['exiftool', '-ver'], stdout=nul, - stderr=nul) - except (OSError, subprocess.CalledProcessError): - print('`perl` (https://www.perl.org/) or `exiftool` ' - '(http://www.sno.phy.queensu.ca/~phil/exiftool/) ' - 'need to be installed and made available in your PATH. ' - 'On Ubuntu, you can install perl with ' - '`sudo apt-get install perl` or install exiftool with ' - '`sudo apt-get install exiftool`.') - sys.exit(1) - - # Check if java is installed. - try: - out = subprocess.check_call(['java', '-version'], stdout=nul, - stderr=nul) - except (OSError, subprocess.CalledProcessError): - print('`java` (https://java.com/) needs to be installed and needs to ' - 'be made available in your PATH. If using Ubuntu, you can do ' - '`sudo apt-get install openjdk-7-jre`') - sys.exit(1) - - # Check if avprobe of ffprobe is installed if system is not Linux. - # If it's linux, we use static builds from http://johnvansickle.com/libav/ - if platform.system() != 'Linux': +setup_deps = [ + setupdeps.Distro(), + setupdeps.SetupTools() +] + +install_deps = [ + # Core deps + setupdeps.LibMagic(), + setupdeps.PythonMagic(), + setupdeps.Six(), + setupdeps.ExifTool(), + setupdeps.AppDirs(), + # Image deps + setupdeps.PathLib(), + setupdeps.LibLZMA(), + setupdeps.LZMA(), + setupdeps.Pillow(), + setupdeps.Numpy(), + setupdeps.Dlib(), + setupdeps.ScikitImage(), + setupdeps.MagickWand(), + setupdeps.Wand(), + setupdeps.LibZBar(), + setupdeps.ZBar(), + setupdeps.JavaJRE(), + setupdeps.PyColorName(), + # Audio video deps + setupdeps.FFProbe(), +] + + +def check_deps(deplist): + failed_deps = [] + for dep in deplist: try: - out = subprocess.check_call(['ffprobe', '-version'], stdout=nul, - stderr=nul) - except (OSError, subprocess.CalledProcessError): - try: - out = subprocess.check_output(['avprobe', '-version'], - stdout=nul, stderr=nul) - except (OSError, subprocess.CalledProcessError): - print('`ffprobe` (https://ffmpeg.org/ffprobe.html) or ' - '`avprobe` (http://libav.org/documentation/avprobe.html)' - 'need to be installed and made available in your PATH. ' - 'If using Ubuntu, you can install avprobe with ' - '`sudo apt-get install libav-tools`.') - sys.exit(1) + log_msg = dep.check() + log.info(dep.name + ' - ' + log_msg) + except setupdeps.CheckFailed as err: + log.info(dep.name + ' - ' + err.args[0]) + failed_deps.append(dep) + + if len(failed_deps) > 0: + msg = 'Some dependencies could not be installed automatically: ' + msg += ", ".join(i.name for i in failed_deps) + for dep in failed_deps: + install_msg = dep.install_help_msg() + if install_msg: + msg += '\n* ' + install_msg + raise DistutilsSetupError(msg) + return deplist + + +def get_install_requires(deplist): + packages = [] + for dep in check_deps(deplist): + packages += dep.get_install_requires() + return packages def read_reqs(reqs_filename): reqs = open(reqs_filename).read().strip().splitlines() return list(i for i in reqs if not (i.startswith('#') or len(i) == 0)) -required = read_reqs('requirements.txt') -test_required = read_reqs('test-requirements.txt') -VERSION = open(os.path.join('file_metadata', 'VERSION')).read().strip() -if sys.version_info >= (3,): - # mock is not required for python 3 - test_required.remove('mock') +if __name__ == '__main__': + log.info('Check and install dependencies required for setup:') + setup_packages = [] + for dep in check_deps(setup_deps): + setup_packages += dep.get_setup_requires() + + if setup_packages: + log.info('Some required packages required for file-metadata\'s setup ' + 'were not found: {0}.\nInstalling them with `pip` ...' + .format(", ".join(setup_packages))) + out = setupdeps.setup_install(setup_packages) + if out is not True: + raise DistutilsSetupError( + 'Unable to install package(s) required by the setup script ' + 'using pip: {0}\nReport this issue to the maintainer.\n' + 'Temporarily, this could be solved by running: ' + '`python -m pip install {1}`.' + .format(", ".join(setup_packages), " ".join(setup_packages))) + + log.info('Check dependencies required for using file-metadata:') + install_required = get_install_requires(install_deps) + test_required = read_reqs('test-requirements.txt') + VERSION = open(os.path.join('file_metadata', 'VERSION')).read().strip() + + if sys.version_info >= (3,): # mock is not required for python 3 + test_required.remove('mock') + + from setuptools import find_packages, setup -if __name__ == "__main__": setup(name='file-metadata', version=VERSION, description='Helps to find structured metadata from a given file.', @@ -78,13 +113,14 @@ def read_reqs(reqs_filename): author_email="dr.trigon@surfeu.ch", maintainer="AbdealiJK", maintainer_email='abdealikothari@gmail.com', + license="MIT", url='https://github.com/AbdealiJK/file-metadata', packages=find_packages(exclude=["build.*", "tests.*", "tests"]), - install_requires=required, + # Dependency management by pip + install_requires=install_required, tests_require=test_required, - license="MIT", # Setuptools has a bug where they use isinstance(x, str) instead - # of basestring. Because of this we convert it to str. + # of basestring. Because of this we convert it to str for Py2. package_data={str('file_metadata'): [str("VERSION")]}, # from http://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ diff --git a/setupdeps.py b/setupdeps.py new file mode 100644 index 0000000..44e5d5a --- /dev/null +++ b/setupdeps.py @@ -0,0 +1,554 @@ +# -*- coding: utf-8 -*- +""" +Various dependencies that are required for file-metadata which need some +special handling. +""" + +from __future__ import (division, absolute_import, unicode_literals, + print_function) + +import ctypes.util +import hashlib +import os +import subprocess +import sys +from distutils import sysconfig + +try: + from urllib.request import urlopen +except ImportError: # Python 2 + from urllib2 import urlopen + + +def which(cmd): + try: + from shutil import which + return which(cmd) + except ImportError: # For python 3.2 and lower + try: + output = subprocess.check_output(["which", cmd], + stderr=subprocess.STDOUT) + except (OSError, subprocess.CalledProcessError): + return None + else: + output = output.decode(sys.getfilesystemencoding()) + return output.strip() + + +def setup_install(packages): + """ + Install packages using pip to the current folder. Useful to import + packages during setup itself. + """ + packages = list(packages) + if not packages: + return True + try: + subprocess.call([sys.executable, "-m", "pip", "install", + "-t", os.getcwd()] + packages) + return True + except subprocess.CalledProcessError: + return False + + +def download(url, filename, overwrite=False, timeout=None, md5=None): + """ + Download the given URL to the given filename. If the file exists, + it won't be downloaded unless asked to overwrite. Both, text data + like html, txt, etc. or binary data like images, audio, etc. are + acceptable. + + :param url: A URL to download. + :param filename: The file to store the downloaded file to. + :param overwrite: Set to True if the file should be downloaded even if it + already exists. + :param md5: The md5 checksum to verify the file using. + """ + blocksize = 16 * 1024 + _hash = hashlib.md5() + if not os.path.exists(filename) or overwrite: + if timeout is None: + response = urlopen(url) + else: + response = urlopen(url, timeout=timeout) + with open(filename, 'wb') as out_file: + while 1: + buf = response.read(blocksize) + if not buf: + break + out_file.write(buf) + _hash.update(buf) + return _hash.hexdigest() == md5 + + +class CheckFailed(Exception): + """ + Exception thrown when a ``SetupPackage.check()`` fails. + """ + pass + + +class PkgConfig(object): + """ + This is a class for communicating with pkg-config. + """ + def __init__(self): + if sys.platform == 'win32': + self.has_pkgconfig = False + else: + self.pkg_config = os.environ.get('PKG_CONFIG', 'pkg-config') + self.set_pkgconfig_path() + + try: + with open(os.devnull) as nul: + subprocess.check_call([self.pkg_config, "--help"], + stdout=nul, stderr=nul) + self.has_pkgconfig = True + except (subprocess.CalledProcessError, OSError): + self.has_pkgconfig = False + print("IMPORTANT WARNING: pkg-config is not installed.") + print("file-metadata may not be able to find some of " + "its dependencies.") + + def set_pkgconfig_path(self): + pkgconfig_path = sysconfig.get_config_var('LIBDIR') + if pkgconfig_path is None: + return + + pkgconfig_path = os.path.join(pkgconfig_path, 'pkgconfig') + if not os.path.isdir(pkgconfig_path): + return + + os.environ['PKG_CONFIG_PATH'] = ':'.join( + [os.environ.get('PKG_CONFIG_PATH', ""), pkgconfig_path]) + + def get_version(self, package): + """ + Get the version of the package from pkg-config. + """ + if not self.has_pkgconfig: + return None + + try: + output = subprocess.check_output( + [self.pkg_config, package, "--modversion"], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + return None + else: + output = output.decode(sys.getfilesystemencoding()) + return output.strip() + + +# The PkgConfig class should be used through this singleton +pkg_config = PkgConfig() + + +class SetupPackage(object): + name = None + optional = False + pkg_names = { + "apt-get": None, + "yum": None, + "dnf": None, + "pacman": None, + "zypper": None, + "brew": None, + "port": None, + "windows_url": None + } + + def check(self): + """ + Check whether the dependencies are met. Should raise a ``CheckFailed`` + exception if the dependency was not found. + """ + pass + + def get_install_requires(self): + """ + Return a list of Python packages that are required by the package. + pip / easy_install will attempt to download and install this + package if it is not installed. + """ + return [] + + def get_setup_requires(self): + """ + Return a list of Python packages that are required by the setup.py + itself. pip / easy_install will attempt to download and install this + package if it is not installed on top of the setup.py script. + """ + return [] + + def install_help_msg(self): + """ + The help message to show if the package is not installed. The help + message shown depends on whether some class variables are present. + """ + def _try_managers(*managers): + for manager in managers: + pkg_name = self.pkg_names.get(manager, None) + if pkg_name and which(manager) is not None: + pkg_note = None + if isinstance(pkg_name, (tuple, list)): + pkg_name, pkg_note = pkg_name + msg = ('Try installing {0} with `{1} install {2}`.' + .format(self.name, manager, pkg_name)) + if pkg_note: + msg += ' Note: ' + pkg_note + return msg + + message = None + if sys.platform == "win32": + url = self.pkg_names.get("windows_url", None) + if url: + return ('Please check {0} for instructions to install {1}' + .format(url, self.name)) + elif sys.platform == "darwin": + manager_message = _try_managers("brew", "port") + return manager_message or message + elif sys.platform.startswith("linux"): + try: + import distro + except ImportError: + setup_install(['distro']) + import distro + release = distro.id() + if release in ('debian', 'ubuntu', 'linuxmint', 'raspbian'): + manager_message = _try_managers('apt-get') + if manager_message: + return manager_message + elif release in ('centos', 'rhel', 'redhat', 'fedora', + 'scientific', 'amazon', ): + manager_message = _try_managers('dnf', 'yum') + if manager_message: + return manager_message + elif release in ('sles', 'opensuse'): + manager_message = _try_managers('zypper') + if manager_message: + return manager_message + elif release in ('arch'): + manager_message = _try_managers('pacman') + if manager_message: + return manager_message + return message + + +class Distro(SetupPackage): + name = "distro" + + def check(self): + return 'Will be installed with pip.' + + def get_setup_requires(self): + try: + import distro # noqa (unused import) + return [] + except ImportError: + return ['distro'] + + +class SetupTools(SetupPackage): + name = 'setuptools' + + def check(self): + return 'Will be installed with pip.' + + def get_setup_requires(self): + try: + import setuptools # noqa (unused import) + return [] + except ImportError: + return ['setuptools'] + + +class PathLib(SetupPackage): + name = 'pathlib' + + def check(self): + if sys.version_info < (3, 4): + return 'Backported pathlib2 will be installed with pip.' + else: + return 'Already installed in python 3.4+' + + def get_install_requires(self): + if sys.version_info < (3, 4): + return ['pathlib2'] + else: + return [] + + +class AppDirs(SetupPackage): + name = 'appdirs' + + def check(self): + return 'Will be installed with pip.' + + def get_install_requires(self): + return ['appdirs'] + + +class LibLZMA(SetupPackage): + name = 'liblzma' + pkg_names = { + "apt-get": 'xz-utils', + "yum": 'xz', + "dnf": 'xz', + "pacman": None, + "zypper": None, + "brew": 'xz', + "port": None, + "windows_url": None + } + + def check(self): + try: + import lzma # noqa (unused import) + return 'Not required as lzma is pre-installed in python.' + except ImportError: + liblzma = pkg_config.get_version('liblzma') + if liblzma is None: + raise CheckFailed('Needs to be installed manually.') + else: + return 'Found with pkg-config.' + + +class LZMA(SetupPackage): + name = 'lzma' + + def check(self): + try: + import lzma # noqa (unused import) + return 'Already installed in python.' + except ImportError: + return 'Will install backports.lzma with pip.' + + def get_install_requires(self): + try: + import lzma # noqa (unused import) + return [] + except ImportError: + return ['backports.lzma'] + + +class LibMagic(SetupPackage): + name = 'libmagic' + pkg_names = { + "apt-get": 'libmagic-dev', + "yum": 'file', + "dnf": 'file', + "pacman": None, + "zypper": None, + "brew": 'libmagic', + "port": None, + "windows_url": None + } + + def check(self): + file_path = which('file') + if file_path is None: + raise CheckFailed('Needs to be installed manually.') + else: + return 'Found "file" utility at {0}.'.format(file_path) + + +class PythonMagic(SetupPackage): + name = 'python-magic' + + def check(self): + return 'Will be installed with pip.' + + def get_install_requires(self): + return ['python-magic'] + + +class Six(SetupPackage): + name = 'six' + + def check(self): + return 'Will be installed with pip.' + + def get_install_requires(self): + return ['six>=1.8.0'] + + +class ExifTool(SetupPackage): + name = 'exiftool' + pkg_names = { + "apt-get": 'libimage-exiftool-perl', + "yum": 'perl-Image-ExifTool', + "dnf": 'perl-Image-ExifTool', + "pacman": None, + "zypper": None, + "brew": 'exiftool', + "port": 'p5-image-exiftool', + "windows_url": 'http://www.sno.phy.queensu.ca/~phil/exiftool/' + } + + def check(self): + exiftool_path = which('exiftool') + if exiftool_path is None: + raise CheckFailed('Needs to be installed manually.') + else: + return 'Found at {0}.'.format(exiftool_path) + + +class Pillow(SetupPackage): + name = 'pillow' + + def check(self): + return 'Will be installed with pip.' + + def get_install_requires(self): + return ['pillow>=2.5.0'] + + +class Numpy(SetupPackage): + name = 'numpy' + + def check(self): + return 'Will be installed with pip.' + + def get_install_requires(self): + return ['numpy>=1.7.2'] + + +class Dlib(SetupPackage): + name = 'dlib' + + def check(self): + return 'Will be installed with pip.' + + def get_install_requires(self): + return ['dlib'] + + +class ScikitImage(SetupPackage): + name = 'scikit-image' + + def check(self): + return 'Will be installed with pip.' + + def get_install_requires(self): + # For some reason matplotlib - a dependency of scikit-image doesn't + # get installed by pip automatically: + # https://github.com/scikit-image/scikit-image/issues/2155 + return ['matplotlib', 'scikit-image>=0.12'] + + +class MagickWand(SetupPackage): + name = 'magickwand' + pkg_names = { + "apt-get": 'libmagickwand-dev', + "yum": 'ImageMagick-devel', + "dnf": 'ImageMagick-devel', + "pacman": None, + "zypper": None, + "brew": 'imagemagick', + "port": 'imagemagick', + "windows_url": ("http://docs.wand-py.org/en/latest/guide/" + "install.html#install-imagemagick-on-windows") + } + + def check(self): + # `wand` already checks for magickwand, but only when importing, not + # during installation. See https://github.com/dahlia/wand/issues/293 + magick_wand = pkg_config.get_version("MagickWand") + if magick_wand is None: + raise CheckFailed('Needs to be installed manually.') + else: + return 'Found with pkg-config.' + + +class Wand(SetupPackage): + name = 'wand' + + def check(self): + return 'Will be installed with pip.' + + def get_install_requires(self): + return ['wand'] + + +class PyColorName(SetupPackage): + name = 'pycolorname' + + def check(self): + return 'Will be installed with pip.' + + def get_install_requires(self): + return ['pycolorname'] + + +class LibZBar(SetupPackage): + name = 'libzbar' + pkg_names = { + "apt-get": 'libzbar-dev', + "yum": 'zbar-devel', + "dnf": 'zbar-devel', + "pacman": None, + "zypper": None, + "brew": 'zbar', + "port": None, + "windows_url": None + } + + def check(self): + libzbar = ctypes.util.find_library('zbar') + if libzbar is None: + raise CheckFailed('Needs to be installed manually.') + else: + return 'Found {0}.'.format(libzbar) + + +class ZBar(SetupPackage): + name = 'zbar' + + def check(self): + return 'Will be installed with pip.' + + def get_install_requires(self): + return ['zbar'] + + +class JavaJRE(SetupPackage): + name = 'java' + pkg_names = { + "apt-get": 'openjdk-7-jre', + "yum": 'java', + "dnf": 'java', + "pacman": None, + "zypper": None, + "brew": None, + "port": None, + "windows_url": "https://java.com/download/" + } + + def check(self): + java_path = which('java') + if java_path is None: + raise CheckFailed('Needs to be installed manually.') + else: + return 'Found at {0}.'.format(java_path) + + +class FFProbe(SetupPackage): + name = 'ffprobe' + pkg_names = { + "apt-get": 'libav-tools', + "yum": ('ffmpeg', 'This requires the RPMFusion repo to be enabled.'), + "dnf": ('ffmpeg', 'This requires the RPMFusion repo to be enabled.'), + "pacman": None, + "zypper": None, + "brew": 'ffmpeg', + "port": None, + "windows_url": None + } + + def check(self): + ffprobe_path = which('ffprobe') or which('avprobe') + if ffprobe_path is None: + raise CheckFailed('Needs to be installed manually.') + else: + return 'Found at {0}.'.format(ffprobe_path)