From dbc4ecf043140780ca0c082c644c5b665f19c443 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Mon, 4 Jul 2016 21:53:48 -0300 Subject: [PATCH] MAINT: Update project boilerplate files improvements. --- .bumpversion.cfg | 34 +++++++++++++++++++ .cookiecutterrc | 6 ++-- .coveragerc | 15 +++++++++ .gitignore | 3 ++ .travis.yml | 12 +++++-- HISTORY.rst | 15 +++++---- MANIFEST.in | 2 +- Makefile | 55 +++++++++++++++++++++--------- README.rst | 21 +++++++++--- VERSION | 1 + docs/conf.py | 24 ++++++------- docs/index.rst | 10 +++++- docs/installation.rst | 2 +- pytest.ini | 13 +++++++ requirements-dev.txt | 8 +++++ requirements-install.txt | 3 ++ requirements-setup.txt | 2 ++ requirements-tests.txt | 5 +++ setup.cfg | 60 --------------------------------- setup.py | 32 +++++++++++------- src/inline_requests/__init__.py | 2 +- tox.ini | 11 +++--- 22 files changed, 207 insertions(+), 129 deletions(-) create mode 100644 .bumpversion.cfg create mode 100644 .coveragerc create mode 100644 VERSION create mode 100644 pytest.ini create mode 100644 requirements-dev.txt create mode 100644 requirements-install.txt create mode 100644 requirements-setup.txt create mode 100644 requirements-tests.txt diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 0000000..630be0a --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,34 @@ +[bumpversion] +current_version = 0.3.0-dev +commit = False +tag = False +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P\w+))? +serialize = + {major}.{minor}.{patch}-{release} + {major}.{minor}.{patch} + +[bumpversion:part:release] +optional_value = placeholder +values = + dev + placeholder + +[bumpversion:file:VERSION] +search = {current_version} +replace = {new_version} + +[bumpversion:file:src/inline_requests/__init__.py] +search = __version__ = '{current_version}' +replace = __version__ = '{new_version}' + +[bumpversion:file:.cookiecutterrc] +search = version: {current_version} +replace = version: {new_version} + +[bumpversion:file:HISTORY.rst] +search = .. comment:: bumpversion marker +replace = .. comment:: bumpversion marker + + {new_version} ({now:%Y-%m-%d}) + ------------------ + diff --git a/.cookiecutterrc b/.cookiecutterrc index f8faef2..0b0b393 100644 --- a/.cookiecutterrc +++ b/.cookiecutterrc @@ -9,9 +9,11 @@ cookiecutter: project_short_description: A decorator for writing coroutine-like spider callbacks. project_slug: scrapy-inline-requests pypi_username: rolando - release_date: '2012-02-03' + use_codecov: y use_cython: n + use_landscape: y use_pypi_deployment_with_travis: n use_pytest: y - version: 0.3.1dev + use_requiresio: y + version: 0.3.1-dev year: 2012-2016 diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..7a14224 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,15 @@ +[paths] +source = + src + +[run] +branch = true +source = + inline_requests + tests +parallel = true + +[report] +show_missing = true +precision = 2 +omit = diff --git a/.gitignore b/.gitignore index 7d44098..2e33e3c 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,6 @@ docs/_build/ # PyBuilder target/ + +# rope-vim +.ropeproject diff --git a/.travis.yml b/.travis.yml index 6396bf3..22fdf24 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,8 @@ before_install: # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors install: - - pip install -U tox twine + - pip install -U tox + - pip install -U coverage - virtualenv --version - easy_install --version - pip --version @@ -30,7 +31,14 @@ install: # command to run tests, e.g. python setup.py test script: - - tox -v + - tox + +after_success: + - | + # Codecov requires a single .coverage and will run 'coverage xml' to + # generate the report. + coverage combine + bash <(curl -s https://codecov.io/bash) after_failure: - more .tox/log/* | cat diff --git a/HISTORY.rst b/HISTORY.rst index f78cc55..2d03339 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,18 +2,19 @@ History ======= -0.3.1dev (next release) ------------------------ +.. comment:: bumpversion marker -* TODO +0.3.1-dev (unreleased) +---------------------- +* Added deprecation about decorating non-spider functions. +* Warn if the callback returns requests with callback or errback set. This + reverts the compability with requests with callbacks. 0.3.0 (2016-06-24) ------------------ - +* ~~Backward incompatible change: Added more restrictions to the request object (no callback/errback).~~ * Cleanup callback/errback attributes before sending back the request to the - generator. This fixes an edge case when using ``request.repalce()``. -* Warn if the callback returns requests with callback or errback set. -* Added deprecation about decorating non-spider functions. + generator. This fixes an edge case when using ``request.replace()``. * Simplified example spider. 0.2.0 (2016-06-23) diff --git a/MANIFEST.in b/MANIFEST.in index 8a244ff..29592d1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,8 +9,8 @@ include *.rst include *.txt include LICENSE +include VERSION include Makefile -include conftest.py global-exclude __pycache__ *.py[cod] global-exclude *.so *.dylib diff --git a/Makefile b/Makefile index 0ddfd28..fae0817 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: clean-so clean-test clean-pyc clean-build clean-docs clean -.PHONY: docs check check-manifest check-setup lint +.PHONY: docs check check-manifest check-setup check-history lint .PHONY: test test-all coverage .PHONY: compile-reqs install-reqs .PHONY: release dist install build-inplace @@ -21,6 +21,7 @@ help: @echo "check - check setup, code style, setup, etc" @echo "check-manifest - check manifest" @echo "check-setup - check setup" + @echo "check-history - check history" @echo "clean - remove all build, test, coverage and Python artifacts" @echo "clean-build - remove build artifacts" @echo "clean-docs - remove docs artifacts" @@ -39,14 +40,20 @@ help: @echo "develop - install package in develop mode" @echo "install - install the package to the active Python's site-packages" -check: check-setup check-manifest lint +check: check-setup check-manifest check-history lint check-setup: + @echo "Checking package metadata (name, description, etc)" python setup.py check --strict --metadata --restructuredtext check-manifest: + @echo "Checking MANIFEST.in" check-manifest --ignore ".*" +check-history: + @echo "Checking latest version in HISTORY" + VERSION=`cat VERSION`; grep "^$${VERSION}\b" HISTORY.rst + clean: clean-build clean-docs clean-pyc clean-test clean-so clean-build: @@ -80,41 +87,39 @@ build-inplace: python setup.py build_ext --inplace develop: clean - python setup.py develop -v + pip install -e . test: develop - py.test -v + py.test test-all: tox -v coverage: develop - coverage run -m pytest + coverage run -m py.test coverage combine coverage report coverage html $(BROWSER) htmlcov/index.html -compile-reqs: - pip-compile -v requirements.in -o requirements.txt - pip-compile -v dev-requirements.in -o dev-requirements.txt - -install-reqs: - pip install -r requirements.txt - pip install -r dev-requirements.txt - -docs: +docs-build: develop rm -f docs/inline_requests.rst rm -f docs/modules.rst sphinx-apidoc -o docs/ src/inline_requests $(MAKE) -C docs clean $(MAKE) -C docs $(SPHINX_BUILD) + +docs: docs-build $(BROWSER) docs/_build/$(SPHINX_BUILD)/index.html servedocs: docs watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . -release: dist +release: clean check dist + git branch | grep '* master' + # Tagging release. + VERSION=`cat VERSION`; git tag -a v$$VERSION + git push --follow-tags twine upload dist/* dist: clean @@ -123,4 +128,22 @@ dist: clean ls -l dist install: clean - python setup.py install + pip install . + +REQUIREMENTS_IN := $(wildcard requirements*.in) +.PHONY: $(REQUIREMENTS_IN) + +requirements%.txt: requirements%.in + pip-compile -v $< -o $@ + +REQUIREMENTS_TXT := $(REQUIREMENTS_IN:.in=.txt) +ifndef REQUIREMENTS_TXT +REQUIREMENTS_TXT := $(wildcard requirements*.txt) +endif + +compile-reqs: $(REQUIREMENTS_TXT) + @test -z "$$REQUIREMENTS_TXT" && echo "No 'requirements*.in' files. Nothing to do" + +install-reqs: + @test -z "$$REQUIREMENTS_TXT" && echo "No 'requirements*.txt' files. Nothing to do" + $(foreach req,$(REQUIREMENTS_TXT),pip install -r $(req);) diff --git a/README.rst b/README.rst index 6b92d9a..9729f81 100644 --- a/README.rst +++ b/README.rst @@ -5,20 +5,33 @@ Scrapy Inline Requests .. image:: https://img.shields.io/pypi/v/scrapy-inline-requests.svg :target: https://pypi.python.org/pypi/scrapy-inline-requests -.. image:: https://img.shields.io/travis/rolando/scrapy-inline-requests.svg - :target: https://travis-ci.org/rolando/scrapy-inline-requests +.. image:: https://img.shields.io/pypi/pyversions/scrapy-inline-requests.svg + :target: https://pypi.python.org/pypi/scrapy-inline-requests .. image:: https://readthedocs.org/projects/scrapy-inline-requests/badge/?version=latest :target: https://readthedocs.org/projects/scrapy-inline-requests/?badge=latest :alt: Documentation Status +.. image:: https://img.shields.io/travis/rolando/scrapy-inline-requests.svg + :target: https://travis-ci.org/rolando/scrapy-inline-requests -A decorator for writing coroutine-like spider callbacks. +.. image:: https://codecov.io/github/rolando/scrapy-inline-requests/coverage.svg?branch=master + :alt: Coverage Status + :target: https://codecov.io/github/rolando/scrapy-inline-requests + +.. image:: https://landscape.io/github/rolando/scrapy-inline-requests/master/landscape.svg?style=flat + :target: https://landscape.io/github/rolando/scrapy-inline-requests/master + :alt: Code Quality Status -Requires ``Scrapy>=1.0`` and supports Python 2.7+ and 3.4+. +.. image:: https://requires.io/github/rolando/scrapy-inline-requests/requirements.svg?branch=master + :alt: Requirements Status + :target: https://requires.io/github/rolando/scrapy-inline-requests/requirements/?branch=master + +A decorator for writing coroutine-like spider callbacks. * Free software: MIT license * Documentation: https://scrapy-inline-requests.readthedocs.org. +* Python versions: 2.7, 3.4+ Quickstart ---------- diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..746bd19 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.3.1-dev diff --git a/docs/conf.py b/docs/conf.py index dc948c1..55b3e48 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,8 +13,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os +import re # If extensions (or modules to document with autodoc) are in another # directory, add these directories to sys.path here. If the directory is @@ -23,15 +23,7 @@ #sys.path.insert(0, os.path.abspath('.')) # Get the project root dir, which is the parent dir of this -cwd = os.getcwd() -project_root = os.path.dirname(cwd) - -# Insert the project root dir as the first element in the PYTHONPATH. -# This lets us ensure that the source package is imported, and that its -# version is used. -sys.path.insert(0, project_root) - -import inline_requests +project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # -- General configuration --------------------------------------------- @@ -40,7 +32,11 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -62,10 +58,10 @@ # for |version| and |release|, also used in various other places throughout # the built documents. # -# The short X.Y version. -version = inline_requests.__version__ # The full version, including alpha/beta/rc tags. -release = inline_requests.__version__ +release = open(os.path.join(project_root, 'VERSION')).read().strip() +# The short X.Y version. +version = re.findall(r'\d+\.\d+\.\d+', release)[0] # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/index.rst b/docs/index.rst index 88fbbae..f41c0c8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -4,7 +4,7 @@ contain the root `toctree` directive. Welcome to Scrapy Inline Requests's documentation! -====================================== +================================================== Contents: @@ -15,3 +15,11 @@ Contents: installation reference history + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/installation.rst b/docs/installation.rst index 4cdf1d0..93c4ed6 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -42,7 +42,7 @@ Once you have a copy of the source, you can install it with: .. code-block:: console - $ python setup.py install + $ pip install -e . .. _Github repo: https://github.com/rolando/scrapy-inline-requests diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..c108c61 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,13 @@ +[pytest] +norecursedirs = + .* + dist + build +python_files = + test_*.py + *_test.py + tests.py +ignore = + setup.py +addopts = + -rxEfsw -v diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..c13985a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,8 @@ +# This packages are requires only for development and release management. +Sphinx +bumpversion +check-manifest +pip-tools +twine +watchdog +wheel diff --git a/requirements-install.txt b/requirements-install.txt new file mode 100644 index 0000000..3609295 --- /dev/null +++ b/requirements-install.txt @@ -0,0 +1,3 @@ +# This packages are required to install and run our package. +Scrapy>=1.0 +six>=1.5 diff --git a/requirements-setup.txt b/requirements-setup.txt new file mode 100644 index 0000000..51c1dd4 --- /dev/null +++ b/requirements-setup.txt @@ -0,0 +1,2 @@ +# This packages are required before running setup (i.e. build commands require +# to import this packages). diff --git a/requirements-tests.txt b/requirements-tests.txt new file mode 100644 index 0000000..4269f08 --- /dev/null +++ b/requirements-tests.txt @@ -0,0 +1,5 @@ +# This packages are required to run all the tests. +coverage +flake8 +pytest +tox diff --git a/setup.cfg b/setup.cfg index a1dbb58..0a78665 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,66 +1,6 @@ -[bumpversion] -# Release procedure -# 1. adds dev suffix -# 2. removes dev suffix. -current_version = 0.3.1dev -commit = True -tag = False -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?Pdev)? -serialize = - {major}.{minor}.{patch}{release} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = placeholder -values = - dev - placeholder - -[bumpversion:file:setup.py] -search = version='{current_version}' -replace = version='{new_version}' - -[bumpversion:file:src/inline_requests/__init__.py] -search = __version__ = '{current_version}' -replace = __version__ = '{new_version}' - -[bumpversion:file:.cookiecutterrc] -search = version: {current_version} -replace = version: {new_version} - [wheel] universal = 1 [flake8] exclude = docs, tests max-line-length = 120 - -[pytest] -norecursedirs = - .* - dist - build -python_files = - test_*.py - *_test.py - tests.py -ignore = - setup.py -addopts = - -rxEfsw - -[coverage:paths] -source = - src - -[coverage:run] -branch = true -source = - inline_requests - tests -parallel = true - -[coverage:report] -show_missing = true -precision = 2 -omit = diff --git a/setup.py b/setup.py index 5cda72e..c30d1b7 100644 --- a/setup.py +++ b/setup.py @@ -1,40 +1,46 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import io from pkgutil import walk_packages from setuptools import setup def find_packages(path): # This method returns packages and subpackages as well. - for _, name, is_pkg in walk_packages([path]): - if is_pkg: - yield name + return [name for _, name, is_pkg in walk_packages([path]) if is_pkg] def read_file(filename): - with open(filename) as fp: - return fp.read() + with io.open(filename) as fp: + return fp.read().strip() -requirements = [ - 'six>=1.5', - 'scrapy>=1.0', -] +def read_rst(filename): + # Ignore unsupported directives by pypi. + content = read_file(filename) + return ''.join(line for line in io.StringIO(content) + if not line.startswith('.. comment::')) + + +def read_requirements(filename): + return [line.strip() for line in read_file(filename).splitlines() + if not line.startswith('#')] + setup( name='scrapy-inline-requests', - version='0.3.1dev', + version=read_file('VERSION'), description="A decorator for writing coroutine-like spider callbacks.", - long_description=read_file('README.rst') + '\n\n' + read_file('HISTORY.rst'), + long_description=read_rst('README.rst') + '\n\n' + read_rst('HISTORY.rst'), author="Rolando Espinoza", author_email='rolando at rmax.io', url='https://github.com/rolando/scrapy-inline-requests', packages=list(find_packages('src')), package_dir={'': 'src'}, + setup_requires=read_requirements('requirements-setup.txt'), + install_requires=read_requirements('requirements-install.txt'), include_package_data=True, - install_requires=requirements, license="MIT", - zip_safe=True, keywords='scrapy-inline-requests', classifiers=[ 'Development Status :: 4 - Beta', diff --git a/src/inline_requests/__init__.py b/src/inline_requests/__init__.py index a6187b6..874e018 100644 --- a/src/inline_requests/__init__.py +++ b/src/inline_requests/__init__.py @@ -11,7 +11,7 @@ __author__ = 'Rolando Espinoza' __email__ = 'rolando at rmax.io' -__version__ = '0.3.1dev' +__version__ = '0.3.1-dev' __all__ = ['inline_requests'] diff --git a/tox.ini b/tox.ini index 7597d46..ced9806 100644 --- a/tox.ini +++ b/tox.ini @@ -3,11 +3,8 @@ envlist = py27, py34, py35, pypy [testenv] deps = - pytest + -rrequirements-setup.txt + -rrequirements-install.txt + -rrequirements-tests.txt commands = - {posargs:py.test} - -; If you want to make tox run the tests with the same versions, create a -; requirements.txt with the pinned versions and uncomment the following lines: -;deps = -; -r{toxinidir}/requirements.txt + {posargs:coverage run -m py.test}