New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Introducing pybuilder #130
Changes from all commits
8de5d92
727233e
ce3f1c8
d6fdc77
b1b517d
14d0252
98ab4ed
78f3889
1e59f2e
2b369ba
df45135
5e81541
388616b
fbf2f8d
2e41f4c
61dd7d7
1e6a1b5
cf4fe92
9167102
808922b
5677383
5242b02
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -79,4 +79,7 @@ target/ | |
*.ipynb.syncdoc | ||
.ipynb_checkpoints | ||
.ipython-daemon.json | ||
.cache | ||
|
||
.idea/ | ||
.pybuilder/ | ||
_SUCCESS |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,16 +1,14 @@ | ||
dist: xenial | ||
language: python | ||
python: | ||
- "3.4" | ||
- "3.5" | ||
- "3.6" | ||
- "3.7" | ||
- "3.8" | ||
- "pypy3" | ||
install: | ||
- pip install --upgrade pip | ||
- "if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then pip install pylzma; fi" | ||
- "pip install .[test]" | ||
- "pip install -r 'requirements.txt' -r 'requirements-build.txt'" | ||
script: | ||
- pylint pysparkling | ||
- python -m pytest -vv | ||
- pylint src/pysparkling | ||
- pyb -X --no-venvs |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,142 @@ | ||
# -*- coding: utf-8 -*- | ||
import logging | ||
from pathlib import Path | ||
|
||
from pybuilder.core import use_plugin, init, before, after, Author | ||
from pybuilder.errors import BuildFailedException | ||
|
||
use_plugin("python.core") | ||
use_plugin("python.flake8") | ||
use_plugin("python.distutils") | ||
# use_plugin("python.unittest") | ||
# use_plugin("python.coverage") | ||
# https://github.com/AlexeySanko/pybuilder_pytest | ||
# use_plugin('pypi:pybuilder_pytest') | ||
# https://github.com/AlexeySanko/pybuilder_pytest_coverage | ||
# use_plugin('pypi:pybuilder_pytest_coverage') | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could we remove this commented code? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would like to keep them there as it was a reminder for the next steps: coverage & pytest within pybuilder. |
||
|
||
|
||
def _get_pysparkling_version() -> str: | ||
"""Returns the pysparkling version based on the one mentioned in the __init__.py file under the package.""" | ||
from pathlib import Path | ||
import re | ||
|
||
with open(Path(__file__).parent / 'src/pysparkling/__init__.py') as fp: | ||
version_line = next(l for l in fp if l.startswith('__version__')) | ||
return re.search(r'=\s*(["\'])(.*)(\1)\s*$', version_line).group(2) | ||
tools4origins marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
|
||
name = "pysparkling" | ||
default_task = "publish" | ||
version = _get_pysparkling_version() | ||
license = 'MIT' | ||
summary = 'Pure Python implementation of the Spark RDD interface.' | ||
description = None | ||
authors = [ | ||
Author(name='Sven Kreiss', email='me@svenkreiss.com'), | ||
Author(name="Erwan Guyomarc'h", email='tools4origins@gmail.com'), | ||
Author(name='Steven Van Ingelgem', email='steven@vaningelgem.be'), | ||
] | ||
maintainers = [ | ||
Author(name="Erwan Guyomarc'h", email='tools4origins@gmail.com'), | ||
] | ||
requires_python = '>= 3.4' | ||
url = 'https://github.com/pysparkling/pysparkling/' | ||
|
||
_logging_root_level: int = logging.root.getEffectiveLevel() | ||
|
||
|
||
@before('run_unit_tests') | ||
def _set_debug_mode(): | ||
logging.root.setLevel('DEBUG') | ||
|
||
|
||
@after('run_unit_tests') | ||
def _set_debug_mode_after(): | ||
logging.root.setLevel(_logging_root_level) | ||
|
||
|
||
@after('package') | ||
def _add_extras_require(project, logger): | ||
indent_size = 4 | ||
encoding = 'utf-8' | ||
|
||
setup_script = Path(project.expand_path("$dir_dist", "setup.py")) | ||
logger.info("Adding 'extras_require' to setup.py") | ||
setup = setup_script.read_text(encoding=encoding).rstrip() | ||
if setup[-1] != ')': | ||
raise BuildFailedException("This setup.py seems to be wrong?") | ||
|
||
# Get the requirements-dev.txt file line by line, ready for insertion. | ||
requirements_dev = '\n'.join( | ||
' '*4*indent_size + "'" + x.strip() + "'," | ||
for x in (Path(__file__).parent / 'requirements-build.txt').read_text().split('\n') | ||
if x | ||
) | ||
|
||
# TODO: find a nicer way to embed this! | ||
new_setup = ( | ||
setup[:-1].rstrip() | ||
+ f""" | ||
extras_require={{ | ||
'hdfs': ['hdfs>=2.0.0'], | ||
'pandas': ['pandas>=0.23.2'], | ||
'performance': ['matplotlib>=1.5.3'], | ||
'streaming': ['tornado>=4.3'], | ||
'test': [ | ||
{requirements_dev} | ||
] | ||
}}, | ||
) | ||
""" | ||
) | ||
|
||
setup_script.write_text(new_setup, encoding=encoding) | ||
|
||
|
||
@init | ||
def set_properties(project): | ||
# Small tweak to project.list_scripts() as that method lists EVERYTHING in the scripts directory. | ||
# and we're only interested in *.py files: | ||
old_project_list_scripts = project.list_scripts | ||
|
||
def _my_list_scripts(): | ||
return [ | ||
filename | ||
for filename in old_project_list_scripts() | ||
if filename.lower().endswith('.py') | ||
] | ||
setattr(project, 'list_scripts', _my_list_scripts) | ||
|
||
project.set_property('dir_source_main_python', 'src/') | ||
project.set_property('dir_source_main_scripts', 'scripts/') | ||
|
||
project.depends_on_requirements(file='requirements.txt') | ||
|
||
project.set_property('distutils_readme_description', True) | ||
project.set_property('distutils_readme_file', 'README.rst') | ||
|
||
project.set_property_if_unset("pytest_extra_args", []) | ||
project.set_property('pytest_coverage_break_build_threshold', 0) # Don't let coverage break the build (for now) | ||
|
||
project.set_property('distutils_console_scripts', []) | ||
project.set_property( | ||
'distutils_classifiers', | ||
[ | ||
'Development Status :: 4 - Beta', | ||
'Intended Audience :: Developers', | ||
'Natural Language :: English', | ||
'License :: OSI Approved :: MIT License', | ||
'Operating System :: OS Independent', | ||
'Programming Language :: Python', | ||
'Programming Language :: Python :: 3.6', | ||
'Programming Language :: Python :: 3.7', | ||
'Programming Language :: Python :: 3.8', | ||
'Programming Language :: Python :: Implementation :: PyPy', | ||
] | ||
) | ||
|
||
|
||
if __name__ == '__main__': | ||
from pybuilder.cli import main | ||
main('-CX', '--no-venvs') |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
[build-system] | ||
requires = ["pybuilder>=0.12.0"] | ||
build-backend = "pybuilder.pep517" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
backports.tempfile==1.0rc1 | ||
cloudpickle>=0.1.0 | ||
futures>=3.0.1 | ||
pylint>=2.3,<2.6 | ||
memory_profiler>=0.47 | ||
pytest | ||
tornado>=4.3 | ||
pybuilder |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
boto>=2.36.0 | ||
future>=0.15 | ||
requests>=2.6.0 | ||
pytz>=2019.3 | ||
python-dateutil>=2.8.0 |
This file was deleted.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is there a reason to stop supporting these versions?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There were some issues with the code + Python 3.5 stopped with full support in mid 2017 iirc, and ended security fixes 2 years later... The earliest Python version still supported is 3.6 (till end of next year).
If you want I will add them again?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
3.5 or 3.4? 3.5 end of life was 2 weeks ago (it has now entered in security mode only)... But yep, still, it is in the past.
I spend a bit of time on my last PR because of some python<3.6 related issues (unordered dict).
There are some stats on the marketshare of python versions here: https://github.com/hugovk/pypi-tools/tree/master/ and... I think I am in favor of dropping support for 3.5 in the new versions of pysparkling. Its marketshare is shrinking (more than I thought), there is a cost to maintain them, and the previous version of pysparkling still supports it