Pybuilder introduction

.gitignore

@@ -79,4 +79,7 @@ target/
 *.ipynb.syncdoc
 .ipynb_checkpoints
 .ipython-daemon.json
-.cache
+
+.idea/
+.pybuilder/
+_SUCCESS

.travis.yml

@@ -10,7 +10,8 @@ python:
 install:
  - pip install --upgrade pip
  - "if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then pip install pylzma; fi"
- - "pip install .[test]"
+ - "pip install -r 'requirements-build.txt'"
 script:
  - pylint pysparkling
- - python -m pytest -vv
+ - pyb -X
+

build.py

@@ -0,0 +1,143 @@
+#   -*- coding: utf-8 -*-
+import logging
+from pathlib import Path
+
+from pybuilder.core import use_plugin, init, before, after, Author
+from pybuilder.errors import BuildFailedException
+
+use_plugin("python.core")
+use_plugin("python.flake8")
+use_plugin("python.distutils")
+# use_plugin("python.unittest")
+# use_plugin("python.coverage")
+# https://github.com/AlexeySanko/pybuilder_pytest
+# use_plugin('pypi:pybuilder_pytest')
+# https://github.com/AlexeySanko/pybuilder_pytest_coverage
+# use_plugin('pypi:pybuilder_pytest_coverage')
+
+
+def _get_pysparkling_version() -> str:
+    """Returns the pysparkling version based on the one mentioned in the __init__.py file under the package."""
+    from pathlib import Path
+    import re
+
+    with open(Path(__file__).parent / 'src/main/python/pysparkling/__init__.py') as fp:
+        version_line = next(l for l in fp if l.startswith('__version__'))
+        return re.search(r'=\s*(["\'])(.*)(\1)\s*$', version_line).group(2)
+
+
+name = "pysparkling"
+default_task = "publish"
+version = _get_pysparkling_version()
+license = 'MIT'
+summary = 'Pure Python implementation of the Spark RDD interface.'
+description = None
+authors = [
+    Author(name='Sven Kreiss', email='me@svenkreiss.com'),
+    Author(name="Erwan Guyomarc'h", email='tools4origins@gmail.com'),
+    Author(name='Steven Van Ingelgem', email='steven@vaningelgem.be'),
+]
+maintainers = [
+    Author(name="Erwan Guyomarc'h", email='tools4origins@gmail.com'),
+]
+requires_python = '>= 3.4'
+url = 'https://github.com/tools4origins/pysparkling'
+
+_logging_root_level: int = logging.root.getEffectiveLevel()
+
+
+@before('run_unit_tests')
+def _set_debug_mode():
+    logging.root.setLevel('DEBUG')
+
+
+@after('run_unit_tests')
+def _set_debug_mode_after():
+    logging.root.setLevel(_logging_root_level)
+
+
+@after('package')
+def _add_extras_require(project, logger):
+    indent_size = 4
+    encoding = 'utf-8'
+
+    setup_script = Path(project.expand_path("$dir_dist", "setup.py"))
+    logger.info("Adding 'extras_require' to setup.py")
+    setup = setup_script.read_text(encoding=encoding).rstrip()
+    if setup[-1] != ')':
+        raise BuildFailedException("This setup.py seems to be wrong?")
+
+    # Get the requirements-dev.txt file line by line, ready for insertion.
+    requirements_dev = '\n'.join(
+        ' '*4*indent_size + "'" + x.strip() + "',"
+        for x in (Path(__file__).parent / 'requirements-build.txt').read_text().split('\n')
+        if x
+    )
+
+    # TODO: find a nicer way to embed this!
+    new_setup = (
+        setup[:-1].rstrip()
+        + f"""
+        extras_require={{
+            'hdfs': ['hdfs>=2.0.0'],
+            'pandas': ['pandas>=0.23.2'],
+            'performance': ['matplotlib>=1.5.3'],
+            'streaming': ['tornado>=4.3'],
+            'test': [
+{requirements_dev}
+            ]
+        }},
+    )
+"""
+    )
+
+    setup_script.write_text(new_setup, encoding=encoding)
+
+
+@init
+def set_properties(project):
+    # Small tweak to project.list_scripts() as that method lists EVERYTHING in the scripts directory.
+    #   and we're only interested in *.py files:
+    old_project_list_scripts = project.list_scripts
+
+    def _my_list_scripts():
+        return [
+            filename
+            for filename in old_project_list_scripts()
+            if filename.lower().endswith('.py')
+        ]
+    setattr(project, 'list_scripts', _my_list_scripts)
+
+    project.depends_on_requirements(file='requirements.txt')
+
+    project.set_property('distutils_readme_description', True)
+    project.set_property('distutils_readme_file', 'README.rst')
+
+    project.set_property_if_unset("pytest_extra_args", [])
+    project.get_property("pytest_extra_args").append("-x")  # Fail on first failing unittest
+    project.set_property('pytest_coverage_break_build_threshold', 0)  # Don't let coverage break the build (for now)
+
+    project.set_property('distutils_console_scripts', [])
+    project.set_property(
+        'distutils_classifiers',
+        [
+            'Development Status :: 4 - Beta',
+            'Intended Audience :: Developers',
+            'Natural Language :: English',
+            'License :: OSI Approved :: MIT License',
+            'Operating System :: OS Independent',
+            'Programming Language :: Python',
+            'Programming Language :: Python :: 3.4',
+            'Programming Language :: Python :: 3.5',
+            'Programming Language :: Python :: 3.6',
+            'Programming Language :: Python :: 3.7',
+            'Programming Language :: Python :: 3.8',
+            'Programming Language :: Python :: 3.9',
+            'Programming Language :: Python :: Implementation :: PyPy',
+        ]
+    )
+
+
+if __name__ == '__main__':
+    from pybuilder.cli import main
+    main('-CX', '--no-venvs')

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["pybuilder>=0.12.0"]
+build-backend = "pybuilder.pep517"

requirements-build.txt

@@ -0,0 +1,8 @@
+backports.tempfile==1.0rc1
+cloudpickle>=0.1.0
+futures>=3.0.1
+pylint>=2.3,<2.6
+memory_profiler>=0.47
+pytest
+tornado>=4.3
+pybuilder

requirements.txt

@@ -0,0 +1,5 @@
+boto>=2.36.0
+future>=0.15
+requests>=2.6.0
+pytz>=2019.3
+python-dateutil>=2.8.0

setup.cfg

@@ -1,6 +1,6 @@
 [flake8]
 ignore = H301
-exclude = venv*,logo,docs,build
+exclude = venv*,logo,docs,build,target
 
 [tool:pytest]
 addopts = --doctest-modules

pysparkling/context.py → src/main/python/pysparkling/context.py

@@ -3,17 +3,17 @@
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
 
-from collections import defaultdict
 import itertools
 import logging
 import pickle
 import struct
 import time
 import traceback
+from collections import defaultdict
 
 from . import __version__ as PYSPARKLING_VERSION
-from .broadcast import Broadcast
 from . import accumulators
+from .broadcast import Broadcast
 from .cache_manager import CacheManager
 from .exceptions import ContextIsLockedException
 from .fileio import File, TextFile

pysparkling/fileio/__init__.py → src/main/python/pysparkling/fileio/__init__.py

@@ -1,7 +1,8 @@
 from __future__ import absolute_import
-# flake8: noqa
 
 from .file import File
 from .textfile import TextFile
 
+# flake8: noqa
+
 __all__ = ['File', 'TextFile']

pysparkling/fileio/codec/bz2.py → src/main/python/pysparkling/fileio/codec/bz2.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import
 
 import bz2
-from io import BytesIO
 import logging
+from io import BytesIO
 
 from .codec import Codec
 

pysparkling/fileio/codec/gz.py → src/main/python/pysparkling/fileio/codec/gz.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import
 
 import gzip
-from io import BytesIO
 import logging
+from io import BytesIO
 
 from .codec import Codec
 

pysparkling/fileio/codec/tar.py → src/main/python/pysparkling/fileio/codec/tar.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import
 
-from io import BytesIO
 import logging
 import tarfile
+from io import BytesIO
 
 from .codec import Codec
 

pysparkling/fileio/codec/zip.py → src/main/python/pysparkling/fileio/codec/zip.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import
 
-from io import BytesIO
 import logging
 import zipfile
+from io import BytesIO
 
 from .codec import Codec
 

pysparkling/fileio/file.py → src/main/python/pysparkling/fileio/file.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 
-from io import BytesIO
 import logging
+from io import BytesIO
 
 from . import codec
 from . import fs

pysparkling/fileio/fs/__init__.py → src/main/python/pysparkling/fileio/fs/__init__.py

@@ -7,7 +7,6 @@
 from .local import Local
 from .s3 import S3
 
-
 __all__ = ['FileSystem', 'GS', 'Hdfs', 'Http', 'Local', 'S3']
 
 

pysparkling/fileio/fs/gs.py → src/main/python/pysparkling/fileio/fs/gs.py

@@ -1,12 +1,12 @@
 from __future__ import absolute_import
 
-from fnmatch import fnmatch
 import logging
+from fnmatch import fnmatch
 from io import BytesIO, StringIO
 
+from .file_system import FileSystem
 from ...exceptions import FileSystemNotSupported
 from ...utils import Tokenizer, parse_file_uri
-from .file_system import FileSystem
 
 log = logging.getLogger(__name__)
 

pysparkling/fileio/fs/hdfs.py → src/main/python/pysparkling/fileio/fs/hdfs.py

@@ -1,12 +1,12 @@
 from __future__ import absolute_import, unicode_literals
 
-from fnmatch import fnmatch
 import logging
+from fnmatch import fnmatch
 from io import BytesIO, StringIO
 
+from .file_system import FileSystem
 from ...exceptions import FileSystemNotSupported
 from ...utils import parse_file_uri, format_file_uri
-from .file_system import FileSystem
 
 log = logging.getLogger(__name__)
 

pysparkling/fileio/fs/local.py → src/main/python/pysparkling/fileio/fs/local.py

@@ -1,13 +1,13 @@
 from __future__ import absolute_import, unicode_literals
 
 import glob
-from fnmatch import fnmatch
 import io
 import logging
 import os
+from fnmatch import fnmatch
 
-from ...utils import Tokenizer
 from .file_system import FileSystem
+from ...utils import Tokenizer
 
 log = logging.getLogger(__name__)