From 35842b67a8eb6190d1a6a57be9c0e3a829746ce7 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 21 Apr 2017 09:01:20 -0500 Subject: [PATCH 1/3] CI setup for manylinux wheels on tagged commit --- .manylinux-install.sh | 21 +++++++++++++++++++++ .manylinux.sh | 9 +++++++++ .travis.yml | 14 ++++++++++++++ 3 files changed, 44 insertions(+) create mode 100755 .manylinux-install.sh create mode 100644 .manylinux.sh diff --git a/.manylinux-install.sh b/.manylinux-install.sh new file mode 100755 index 0000000..fd0ec23 --- /dev/null +++ b/.manylinux-install.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e -x + +# Compile wheels +for PYBIN in /opt/python/*/bin; do + if [[ "${PYBIN}" == *"cp27"* ]] || \ + [[ "${PYBIN}" == *"cp33"* ]] || \ + [[ "${PYBIN}" == *"cp34"* ]] || \ + [[ "${PYBIN}" == *"cp35"* ]] || \ + [[ "${PYBIN}" == *"cp36"* ]]; then + "${PYBIN}/pip" install -e /io/ + "${PYBIN}/pip" wheel /io/ -w wheelhouse/ + fi +done + +# Bundle external shared libraries into the wheels +for whl in wheelhouse/zope.index*.whl; do + auditwheel repair "$whl" -w /io/wheelhouse/ +done + diff --git a/.manylinux.sh b/.manylinux.sh new file mode 100644 index 0000000..037a3b2 --- /dev/null +++ b/.manylinux.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e -x + +docker pull $DOCKER_IMAGE + +docker run --rm -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/.manylinux-install.sh + +pip install twine && twine upload -u zope.wheelbuilder -p $PYPIPASSWORD wheelhouse/* diff --git a/.travis.yml b/.travis.yml index e119099..ec7f2ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,20 @@ matrix: - os: osx language: generic env: TERRYFY_PYTHON='macpython 3.5' + - services: + - docker + env: DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64 + before_install: + - if [[ $TRAVIS_TAG ]]; then bash .manylinux.sh; fi + - exit 0 + - services: + - docker + env: + - DOCKER_IMAGE=quay.io/pypa/manylinux1_i686 + - PRE_CMD=linux32 + before_install: + - if [[ $TRAVIS_TAG ]]; then bash .manylinux.sh; fi + - exit 0 before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then git clone https://github.com/MacPython/terryfy; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then source terryfy/travis_tools.sh; fi From dabaac58f3c2dd457228af2b6d08807ef61056e3 Mon Sep 17 00:00:00 2001 From: Jason Madden Date: Fri, 21 Apr 2017 15:05:43 -0500 Subject: [PATCH 2/3] Add support for Python 3.6 Only pass re.LOCALE under Python 2. Fixes #8 --- .travis.yml | 9 ++++-- CHANGES.rst | 16 +++++++--- setup.py | 48 +++++++++++++++++------------ src/zope/index/text/htmlsplitter.py | 14 +++++++-- tox.ini | 9 ++---- 5 files changed, 61 insertions(+), 35 deletions(-) diff --git a/.travis.yml b/.travis.yml index e119099..855179a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,8 @@ matrix: python: 3.4 - os: linux python: 3.5 + - os: linux + python: 3.6 - os: linux python: pypy - os: linux @@ -23,15 +25,18 @@ matrix: - os: osx language: generic env: TERRYFY_PYTHON='macpython 3.5' + - os: osx + language: generic + env: TERRYFY_PYTHON='macpython 3.6.1' before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then git clone https://github.com/MacPython/terryfy; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then source terryfy/travis_tools.sh; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then get_python_environment $TERRYFY_PYTHON venv; fi - pip install --upgrade pip install: - - pip install -e . + - pip install -e .[test] script: - - python setup.py -q test -q + - zope-testrunner --test-path=src notifications: email: false after_success: diff --git a/CHANGES.rst b/CHANGES.rst index 9d5dcda..5dfb85d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,13 @@ Changes 4.2.1 (unreleased) ------------------ -- None are now valid values in a field index. This requires BTrees >= 4.4.1 +- ``None`` are now valid values in a field index. This requires BTrees + >= 4.4.1. +- Allow ``TypeError`` to propagate from a field index when the value + cannot be stored in a BTree. Previously it was silently ignored + because it was expected that these were usually ``None``. +- Add support for Python 3.6. See `issue 8 + `_. 4.2.0 (2016-06-10) ------------------ @@ -132,13 +138,13 @@ Changes This makes it more compatible with other indexes (for example, when using in catalog). This change can lead to problems, if your code somehow depends on the II nature of sets, as it was before. - + Also, FilteredSets used to use IFSets as well, if you have any FilteredSets pickled in the database, you need to migrate them to IFSets yourself. You can do it like that: - + filter._ids = filter.family.IF.Set(filter._ids) - + Where ``filter`` is an instance of FilteredSet. - IMPORTANT: KeywordIndex are now non-normalizing. Because @@ -146,7 +152,7 @@ Changes doesn't make any sense. Instead, it provides the ``normalize`` method that can be overriden by subclasses to provide some normalization. - + The CaseInsensitiveKeywordIndex class is now provided that do case-normalization for string-based keywords. The old CaseSensitiveKeywordIndex is gone, applications should use diff --git a/setup.py b/setup.py index 1d25bdb..b64ba0d 100644 --- a/setup.py +++ b/setup.py @@ -90,35 +90,43 @@ def alltests(): description="Indices for using with catalog like text, field, etc.", long_description=long_description, classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Zope Public License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Topic :: Internet :: WWW/HTTP', - 'Topic :: Software Development', - ], + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Zope Public License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Software Development', + ], packages=find_packages('src'), package_dir = {'': 'src'}, namespace_packages=['zope',], extras_require={ - 'test': [], - 'tools': ['ZODB', 'transaction']}, + 'test': [ + 'zope.testrunner', + ], + 'tools': [ + 'ZODB', + 'transaction' + ] + }, install_requires=[ 'persistent', 'BTrees>=4.4.1', 'setuptools', 'six', - 'zope.interface'], + 'zope.interface' + ], tests_require = ['zope.testrunner'], test_suite = '__main__.alltests', ext_modules=[ diff --git a/src/zope/index/text/htmlsplitter.py b/src/zope/index/text/htmlsplitter.py index 464ab40..c657b53 100644 --- a/src/zope/index/text/htmlsplitter.py +++ b/src/zope/index/text/htmlsplitter.py @@ -20,8 +20,18 @@ from zope.index.text.interfaces import ISplitter MARKUP = re.compile(r"(<[^<>]*>|&[A-Za-z]+;)") -WORDS = re.compile(r"(?L)\w+") -GLOBS = re.compile(r"(?L)\w+[\w*?]*") + +_flags = 0 +if bytes is str: + # On python 2, we want locale aware splitting. This is the default + # on Python 3 when the pattern is text/unicode and in fact is + # forbidden unless the pattern is bytes (starting) in 3.6 + _flags = re.LOCALE + +WORDS = re.compile(r"\w+", _flags) +GLOBS = re.compile(r"\w+[\w*?]*", _flags) + +del _flags @implementer(ISplitter) class HTMLWordSplitter(object): diff --git a/tox.ini b/tox.ini index e0ceb6a..b37a06d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,9 @@ [tox] envlist = - py27,py33,py34,py35,pypy,pypy3 + py27,py33,py34,py35,py36,pypy,pypy3 [testenv] commands = - python setup.py -q test -q + zope-testrunner --test-path=src deps = - BTrees - persistent - zope.interface - zope.testrunner + .[test] From eef4e444a7691299fccfe6eee4e385fe2d7b19e1 Mon Sep 17 00:00:00 2001 From: Jason Madden Date: Fri, 21 Apr 2017 15:33:38 -0500 Subject: [PATCH 3/3] Make the C implementation of okascore importable under Python 3. Fixes #14 Add PURE_PYTHON support and testing for the score function. --- .travis.yml | 3 ++ CHANGES.rst | 4 +++ src/zope/index/text/okapiindex.py | 18 ++++++++-- src/zope/index/text/okascore.c | 35 +++++++++++++++++--- src/zope/index/text/tests/test_okapiindex.py | 14 +++++--- 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 855179a..d69a232 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,9 @@ matrix: include: - os: linux python: 2.7 + - os: linux + python: 2.7 + env: PURE_PYTHON=1 - os: linux python: 3.3 - os: linux diff --git a/CHANGES.rst b/CHANGES.rst index 5dfb85d..86dd5da 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,6 +11,10 @@ Changes because it was expected that these were usually ``None``. - Add support for Python 3.6. See `issue 8 `_. +- Make the C implementation of the text index's score function + (``zope.text.index.okascore``) importable under Python 3. Previously + we would fall back to a pure-Python implementation. See `issue 14 + `_. 4.2.0 (2016-06-10) ------------------ diff --git a/src/zope/index/text/okapiindex.py b/src/zope/index/text/okapiindex.py index 084b30c..c48c438 100644 --- a/src/zope/index/text/okapiindex.py +++ b/src/zope/index/text/okapiindex.py @@ -189,6 +189,9 @@ regardless of k3's value. So, in a trivial sense, we are incorporating this measure (and optimizing it by not bothering to multiply by 1 ). """ +import os +import platform + from zope.index.text.baseindex import BaseIndex from zope.index.text.baseindex import inverse_doc_frequency try: @@ -197,6 +200,15 @@ score = None from BTrees.Length import Length + +_py_impl = getattr(platform, 'python_implementation', lambda: None) +_is_pypy = _py_impl() == 'PyPy' +PURE_PYTHON = os.environ.get('PURE_PYTHON') or _is_pypy +if PURE_PYTHON: + score = None + +PY2 = str is bytes + class OkapiIndex(BaseIndex): # BM25 free parameters. @@ -253,7 +265,7 @@ def _change_doc_len(self, delta): # D to TF(D,t)*IDF(t) directly, where the product is computed as a float. # NOTE: This may be overridden below, by a function that computes the # same thing but with the inner scoring loop in C. - if score is None: #pragma NO COVERAGE + if score is None: def _search_wids(self, wids): if not wids: return [] @@ -330,7 +342,8 @@ def _search_wids(self, wids): d2f = self._wordinfo[t] # map {docid -> f(docid, t)} idf = inverse_doc_frequency(len(d2f), N) # an unscaled float result = self.family.IF.Bucket() - score(result, d2f.items(), docid2len, idf, meandoclen) + items = d2f.items() if PY2 else list(d2f.items()) + score(result, items, docid2len, idf, meandoclen) L.append((result, 1)) return L @@ -358,4 +371,3 @@ def _get_frequencies(self, wids): for wid in wids: d[wid] = dget(wid, 0) + 1 return d, len(wids) - diff --git a/src/zope/index/text/okascore.c b/src/zope/index/text/okascore.c index a0ff0dd..edfcdc2 100644 --- a/src/zope/index/text/okascore.c +++ b/src/zope/index/text/okascore.c @@ -35,6 +35,10 @@ #include "Python.h" +#if PY_MAJOR_VERSION >= 3 +#define PY3K +#endif + #define K1 1.2 #define B 0.75 @@ -128,11 +132,32 @@ static PyMethodDef okascore_functions[] = { {NULL} }; -void -initokascore(void) +#ifdef PY3K +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "okascore", /* m_name */ + "inner scoring loop for Okapi rank", /* m_doc */ + -1, /* m_size */ + okascore_functions, /* m_methods */ + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL, /* m_free */ + }; + +PyMODINIT_FUNC +PyInit_okascore(void) { PyObject *m; - - m = Py_InitModule3("okascore", okascore_functions, - "inner scoring loop for Okapi rank"); + m = PyModule_Create(&moduledef); + return m; } +#else +PyMODINIT_FUNC +initokascore(void) +{ + /* XXX: Error checking */ + Py_InitModule3("okascore", okascore_functions, + "inner scoring loop for Okapi rank"); +} +#endif diff --git a/src/zope/index/text/tests/test_okapiindex.py b/src/zope/index/text/tests/test_okapiindex.py index cd2b4a1..8b9ae63 100644 --- a/src/zope/index/text/tests/test_okapiindex.py +++ b/src/zope/index/text/tests/test_okapiindex.py @@ -150,8 +150,14 @@ def _getBTreesFamily(self): import BTrees return BTrees.family64 +class TestScore(unittest.TestCase): + + def test_score_extension(self): + from zope.index.text.okapiindex import PURE_PYTHON, score + if PURE_PYTHON: + self.assertIsNone(score) + else: + self.assertIsNotNone(score) + def test_suite(): - return unittest.TestSuite(( - unittest.makeSuite(OkapiIndexTest32), - unittest.makeSuite(OkapiIndexTest64), - )) + return unittest.defaultTestLoader.loadTestsFromName(__name__)