From 4f0ea85b737c9c811dbfd6a4733d96974774d0ea Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 12:00:30 -0800 Subject: [PATCH 01/35] further removed Python 3.4 support --- .travis.yml | 1 - HISTORY.rst | 2 ++ pyproject.toml | 2 +- setup.py | 3 +-- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 924a80090..d4fb75e3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python python: - 2.7 - - 3.4 - 3.5 - 3.6 diff --git a/HISTORY.rst b/HISTORY.rst index f64275d86..fe050e849 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,8 @@ Version 0.4.1 ... Changes: +- Support for Python 3.4 was removed. (3.4 reached end-of-life on March 18, + 2019) - Fuzzy intersections were corrected to avoid over-counting partial intersection instances. - Levenshtein can now return an optimal alignment diff --git a/pyproject.toml b/pyproject.toml index 528eb6b0d..2503d0b7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ classifiers = [ ] [tool.poetry.dependencies] -python = "~2.7 || ^3.4" +python = "~2.7 || ^3.5" numpy = ">=1.7" six = ">=1.3" diff --git a/setup.py b/setup.py index 8fbaf0e7f..66536feb4 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,6 @@ def readfile(fn): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', @@ -106,5 +105,5 @@ def readfile(fn): 'pyliblzma>=0.5.3,<0.6.0' ] }, - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.* !=3.3.* !=3.4.*', ) From 8ce3e76f7be5208d40b66edd829d160ccb9db739 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 12:04:27 -0800 Subject: [PATCH 02/35] corrected formatting --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 66536feb4..db22829ed 100644 --- a/setup.py +++ b/setup.py @@ -105,5 +105,5 @@ def readfile(fn): 'pyliblzma>=0.5.3,<0.6.0' ] }, - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.* !=3.3.* !=3.4.*', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', ) From e55da234e50c7346963a658eefff4ee7632ffe6c Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 12:11:26 -0800 Subject: [PATCH 03/35] omit requirements-dev from installation --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d4fb75e3f..0b2635c22 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,6 @@ install: - case "$TRAVIS_PYTHON_VERSION" in 3.5|3.6|3.7) travis_retry pip install paq lzss;; esac - travis_retry pip install coveralls - travis_retry pip install -r requirements.txt - - travis_retry pip install -r requirements-dev.txt - travis_retry python setup.py install # Run test From a740abd145a92f65124c3f747c39deb4e8fbbfa9 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 12:14:18 -0800 Subject: [PATCH 04/35] omit requirements-dev from installation --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b6946dc65..08e6c897b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,7 +26,7 @@ jobs: versionSpec: '$(python.version)' architecture: 'x64' - - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install -r requirements-dev.txt + - script: python -m pip install --upgrade pip && pip install -r requirements.txt displayName: 'Install dependencies' - script: | @@ -68,7 +68,7 @@ jobs: versionSpec: '$(python.version)' architecture: 'x64' - - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install -r requirements-dev.txt + - script: python -m pip install --upgrade pip && pip install -r requirements.txt displayName: 'Install dependencies' - script: | @@ -111,7 +111,7 @@ jobs: versionSpec: '$(python.version)' architecture: 'x64' - - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install -r requirements-dev.txt + - script: python -m pip install --upgrade pip && pip install -r requirements.txt displayName: 'Install dependencies' - script: | From e6916a9907bef8cc31da00a4867e0b4e52816536 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 12:20:14 -0800 Subject: [PATCH 05/35] added nltk & syllabipy back to CI builds --- .travis.yml | 1 + azure-pipelines.yml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0b2635c22..507654974 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ install: - case "$TRAVIS_PYTHON_VERSION" in 3.5|3.6|3.7) travis_retry pip install paq lzss;; esac - travis_retry pip install coveralls - travis_retry pip install -r requirements.txt + - travis_retry pip install nltk syllabipy - travis_retry python setup.py install # Run test diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 08e6c897b..c4271137b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,7 +26,7 @@ jobs: versionSpec: '$(python.version)' architecture: 'x64' - - script: python -m pip install --upgrade pip && pip install -r requirements.txt + - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install nltk syllabipy displayName: 'Install dependencies' - script: | @@ -68,7 +68,7 @@ jobs: versionSpec: '$(python.version)' architecture: 'x64' - - script: python -m pip install --upgrade pip && pip install -r requirements.txt + - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install nltk syllabipy displayName: 'Install dependencies' - script: | @@ -111,7 +111,7 @@ jobs: versionSpec: '$(python.version)' architecture: 'x64' - - script: python -m pip install --upgrade pip && pip install -r requirements.txt + - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install nltk syllabipy displayName: 'Install dependencies' - script: | From 310f000c81eb380c0bb66c7050b4f9b5f650cf90 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 12:26:14 -0800 Subject: [PATCH 06/35] added requirements-test.txt to list deps for CI tests --- .circleci/config.yml | 2 +- .travis.yml | 2 +- azure-pipelines.yml | 6 +++--- requirements-test.txt | 5 +++++ 4 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 requirements-test.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index 14c1f1a9c..c4d97fcf4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ jobs: . venv/bin/activate echo "tox" >> requirements.txt pip install -r requirements.txt - pip install -r requirements-dev.txt + pip install -r requirements-test.txt - save_cache: paths: diff --git a/.travis.yml b/.travis.yml index 507654974..6fa1f1c7f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ install: - case "$TRAVIS_PYTHON_VERSION" in 3.5|3.6|3.7) travis_retry pip install paq lzss;; esac - travis_retry pip install coveralls - travis_retry pip install -r requirements.txt - - travis_retry pip install nltk syllabipy + - travis_retry pip install -r requirements-test.txt - travis_retry python setup.py install # Run test diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c4271137b..79b95b993 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,7 +26,7 @@ jobs: versionSpec: '$(python.version)' architecture: 'x64' - - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install nltk syllabipy + - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install -r requirements-test.txt displayName: 'Install dependencies' - script: | @@ -68,7 +68,7 @@ jobs: versionSpec: '$(python.version)' architecture: 'x64' - - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install nltk syllabipy + - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install -r requirements-test.txt displayName: 'Install dependencies' - script: | @@ -111,7 +111,7 @@ jobs: versionSpec: '$(python.version)' architecture: 'x64' - - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install nltk syllabipy + - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install -r requirements-test.txt displayName: 'Install dependencies' - script: | diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 000000000..9dda146fd --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,5 @@ +# Library dependencies for the Python code. You need to install these with +# `pip install -r requirements-test.txt` before you can run this. + +nltk +syllabipy From f675092d26e2a635191bd7d4e411533431b45560 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 12:44:45 -0800 Subject: [PATCH 07/35] skip testing Chao distances for Python <= 3.5 --- tests/distance/test_distance_chao_dice.py | 9 +++++++++ tests/distance/test_distance_chao_jaccard.py | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/tests/distance/test_distance_chao_dice.py b/tests/distance/test_distance_chao_dice.py index 728e3d645..683d2de6d 100644 --- a/tests/distance/test_distance_chao_dice.py +++ b/tests/distance/test_distance_chao_dice.py @@ -29,6 +29,7 @@ ) import random +import sys import unittest from abydos.distance import ChaoDice @@ -44,6 +45,10 @@ class ChaoDiceTestCases(unittest.TestCase): def test_chao_dice_sim(self): """Test abydos.distance.ChaoDice.sim.""" + # Skip testing for Python <= 3.5 + if sys.version_info[0:2] < (3, 6): + return + random.seed(0) # Base cases @@ -66,6 +71,10 @@ def test_chao_dice_sim(self): def test_chao_dice_sim_score(self): """Test abydos.distance.ChaoDice.sim_score.""" + # Skip testing for Python <= 3.5 + if sys.version_info[0:2] < (3, 6): + return + random.seed(0) # Base cases diff --git a/tests/distance/test_distance_chao_jaccard.py b/tests/distance/test_distance_chao_jaccard.py index 12dfcf446..6fd39aa98 100644 --- a/tests/distance/test_distance_chao_jaccard.py +++ b/tests/distance/test_distance_chao_jaccard.py @@ -29,6 +29,7 @@ ) import random +import sys import unittest from abydos.distance import ChaoJaccard @@ -44,6 +45,9 @@ class ChaoJaccardTestCases(unittest.TestCase): def test_chao_jaccard_sim(self): """Test abydos.distance.ChaoJaccard.sim.""" + # Skip testing for Python <= 3.5 + if sys.version_info[0:2] < (3, 6): + return random.seed(0) # Base cases @@ -66,6 +70,10 @@ def test_chao_jaccard_sim(self): def test_chao_jaccard_sim_score(self): """Test abydos.distance.ChaoJaccard.sim_score.""" + # Skip testing for Python <= 3.5 + if sys.version_info[0:2] < (3, 6): + return + random.seed(0) # Base cases From 9135b7b46ae64adb7bc8f1d97fe05e40ad2f59a3 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 12:50:40 -0800 Subject: [PATCH 08/35] added 3.7 & 3.8 builds & xenial by default --- .travis.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6fa1f1c7f..a6cee0acb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,12 @@ language: python +dist: xenial python: - 2.7 - 3.5 - 3.6 - -matrix: - include: - - python: 3.7 - dist: xenial - sudo: true + - 3.7 + - 3.8 notifications: email: false @@ -17,8 +14,8 @@ notifications: # Install packages install: - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install pyliblzma; fi - - case "$TRAVIS_PYTHON_VERSION" in 3.5|3.6|3.7) travis_retry pip install cython;; esac - - case "$TRAVIS_PYTHON_VERSION" in 3.5|3.6|3.7) travis_retry pip install paq lzss;; esac + - case "$TRAVIS_PYTHON_VERSION" in 3.5|3.6|3.7|3.8) travis_retry pip install cython;; esac + - case "$TRAVIS_PYTHON_VERSION" in 3.5|3.6|3.7|3.8) travis_retry pip install paq lzss;; esac - travis_retry pip install coveralls - travis_retry pip install -r requirements.txt - travis_retry pip install -r requirements-test.txt From ded7fbfcef36286f15c13e0eba120f18c0004b78 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 12:55:11 -0800 Subject: [PATCH 09/35] added py36 option back for semaphore --- tox.ini | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 2e1524916..8c6b85c8e 100644 --- a/tox.ini +++ b/tox.ini @@ -27,7 +27,7 @@ deps = coverage py27: pyliblzma lzss - py37,py37,doctest: paq + py37,py36,doctest: paq nltk syllabipy commands = nosetests [] @@ -59,13 +59,17 @@ commands = {[testenv:regression]commands} deps = {[testenv:regression]deps} commands = {[testenv:regression]commands} +[testenv:py36-regression] +deps = {[testenv:regression]deps} +commands = {[testenv:regression]commands} + [testenv:fuzz] deps = nose nltk syllabipy - py37: lzss - py37: paq + py37, py36: lzss + py37, py36: paq commands = nosetests {toxinidir}/tests/fuzz --processes=-1 \ --process-timeout=1200 --process-restartworker -c=0 -v [] @@ -77,6 +81,9 @@ commands = {[testenv:fuzz]commands} deps = {[testenv:fuzz]deps} commands = {[testenv:fuzz]commands} +[testenv:py36-fuzz] +deps = {[testenv:fuzz]deps} +commands = {[testenv:fuzz]commands} [testenv:pylint] basepython = python3.7 From eb316ee41cf13422b8e7816bd388f13130bb0b66 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 14:04:53 -0800 Subject: [PATCH 10/35] combined matrix for all pythons/platforms --- azure-pipelines.yml | 136 +++++++++++++------------------------------- 1 file changed, 38 insertions(+), 98 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 79b95b993..6dddfb91d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,106 +5,60 @@ jobs: -- job: 'Test_Linux' - pool: - vmImage: 'ubuntu-16.04' +- job: 'Test' + strategy: matrix: - Python27: + Python27Linux: python.version: '2.7' - Python35: + imageName: 'ubuntu-16.04' + Python35Linux: python.version: '3.5' - Python36: + imageName: 'ubuntu-16.04' + Python36Linux: python.version: '3.6' - Python37: + imageName: 'ubuntu-16.04' + Python37Linux: python.version: '3.7' - maxParallel: 4 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' - - - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install -r requirements-test.txt - displayName: 'Install dependencies' - - - script: | - pip install pytest - pip install pytest-cov - pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=abydos --cov-report=xml --cov-report=html - displayName: 'pytest' - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() - - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' - reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' - -- job: 'Test_Windows' - pool: - vmImage: 'vs2017-win2016' - strategy: - matrix: - Python27: + imageName: 'ubuntu-16.04' + Python38Linux: + python.version: '3.8' + imageName: 'ubuntu-16.04' + Python27Windows: python.version: '2.7' - Python35: + imageName: 'vs2017-win2016' + Python35Windows: python.version: '3.5' - Python36: + imageName: 'vs2017-win2016' + Python36Windows: python.version: '3.6' - Python37: + imageName: 'vs2017-win2016' + Python37Windows: python.version: '3.7' - maxParallel: 4 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - architecture: 'x64' - - - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install -r requirements-test.txt - displayName: 'Install dependencies' - - - script: | - pip install pytest - pip install pytest-cov - pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov=abydos --cov-report=xml --cov-report=html - displayName: 'pytest' - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() - - - task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' - reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' - - -- job: 'Test_MacOS' - pool: - vmImage: 'macOS-10.13' - strategy: - matrix: - Python27: + imageName: 'vs2017-win2016' + Python38Windows: + python.version: '3.8' + imageName: 'vs2017-win2016' + Python27MacOS: python.version: '2.7' - Python35: + imageName: 'macos-10.13' + Python35MacOS: python.version: '3.5' - Python36: + imageName: 'macos-10.13' + Python36MacOS: python.version: '3.6' - Python37: + imageName: 'macos-10.13' + Python37MacOS: python.version: '3.7' + imageName: 'macos-10.13' + Python38MacOS: + python.version: '3.8' + imageName: 'macos-10.13' maxParallel: 4 + pool: + vmImage: $(imageName) + steps: - task: UsePythonVersion@0 inputs: @@ -131,17 +85,3 @@ jobs: codeCoverageTool: Cobertura summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov' - -- job: 'Publish' - dependsOn: 'Test_Linux' - pool: - vmImage: 'Ubuntu 16.04' - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.x' - architecture: 'x64' - - - script: python setup.py sdist - displayName: 'Build sdist' From 895c831403b47c5d007c8adaa22a2a094b49df13 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 14:12:17 -0800 Subject: [PATCH 11/35] bumped OS images to -latest --- azure-pipelines.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6dddfb91d..7f12f0e1a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,49 +11,49 @@ jobs: matrix: Python27Linux: python.version: '2.7' - imageName: 'ubuntu-16.04' + imageName: 'ubuntu-latest' Python35Linux: python.version: '3.5' - imageName: 'ubuntu-16.04' + imageName: 'ubuntu-latest' Python36Linux: python.version: '3.6' - imageName: 'ubuntu-16.04' + imageName: 'ubuntu-latest' Python37Linux: python.version: '3.7' - imageName: 'ubuntu-16.04' + imageName: 'ubuntu-latest' Python38Linux: python.version: '3.8' - imageName: 'ubuntu-16.04' + imageName: 'ubuntu-latest' Python27Windows: python.version: '2.7' - imageName: 'vs2017-win2016' + imageName: 'windows-latest' Python35Windows: python.version: '3.5' - imageName: 'vs2017-win2016' + imageName: 'windows-latest' Python36Windows: python.version: '3.6' - imageName: 'vs2017-win2016' + imageName: 'windows-latest' Python37Windows: python.version: '3.7' - imageName: 'vs2017-win2016' + imageName: 'windows-latest' Python38Windows: python.version: '3.8' - imageName: 'vs2017-win2016' + imageName: 'windows-latest' Python27MacOS: python.version: '2.7' - imageName: 'macos-10.13' + imageName: 'macOS-latest' Python35MacOS: python.version: '3.5' - imageName: 'macos-10.13' + imageName: 'macOS-latest' Python36MacOS: python.version: '3.6' - imageName: 'macos-10.13' + imageName: 'macOS-latest' Python37MacOS: python.version: '3.7' - imageName: 'macos-10.13' + imageName: 'macOS-latest' Python38MacOS: python.version: '3.8' - imageName: 'macos-10.13' + imageName: 'macOS-latest' maxParallel: 4 pool: From c99b40b4492da136b82b15c775099f45b6578d5e Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 14:27:02 -0800 Subject: [PATCH 12/35] broke out individual keys for testing to pass on non-Linux platforms --- tests/tokenizer/test_tokenizer_q_skipgrams.py | 824 +++++++++--------- 1 file changed, 418 insertions(+), 406 deletions(-) diff --git a/tests/tokenizer/test_tokenizer_q_skipgrams.py b/tests/tokenizer/test_tokenizer_q_skipgrams.py index 805657c9d..54518b9f0 100644 --- a/tests/tokenizer/test_tokenizer_q_skipgrams.py +++ b/tests/tokenizer/test_tokenizer_q_skipgrams.py @@ -134,304 +134,311 @@ def test_qskipgrams(self): sorted(QSkipgrams(qval=1).tokenize('NEILSEN').get_list()), sorted(['N', 'E', 'I', 'L', 'S', 'E', 'N']), ) - self.assertEqual( + + test_counter = ( QSkipgrams(qval=(2,), scaler='SSK') .tokenize('NEILSEN') - .get_counter(), - Counter( - { - '$N': 1.2404672100000003, - '$E': 1.2072969000000002, - '$I': 0.6561, - '$L': 0.5904900000000001, - '$S': 0.531441, - '$#': 0.3874204890000001, - 'NE': 1.341441, - 'NI': 0.7290000000000001, - 'NL': 0.6561, - 'NS': 0.5904900000000001, - 'NN': 0.4782969000000001, - 'N#': 1.2404672100000003, - 'EI': 0.81, - 'EL': 0.7290000000000001, - 'ES': 0.6561, - 'EE': 0.5904900000000001, - 'EN': 1.341441, - 'E#': 1.2072969000000002, - 'IL': 0.81, - 'IS': 0.7290000000000001, - 'IE': 0.6561, - 'IN': 0.5904900000000001, - 'I#': 0.531441, - 'LS': 0.81, - 'LE': 0.7290000000000001, - 'LN': 0.6561, - 'L#': 0.5904900000000001, - 'SE': 0.81, - 'SN': 0.7290000000000001, - 'S#': 0.6561, - } - ), + .get_counter() ) - self.assertEqual( + gold_counter = Counter( + { + '$N': 1.2404672100000003, + '$E': 1.2072969000000002, + '$I': 0.6561, + '$L': 0.5904900000000001, + '$S': 0.531441, + '$#': 0.3874204890000001, + 'NE': 1.341441, + 'NI': 0.7290000000000001, + 'NL': 0.6561, + 'NS': 0.5904900000000001, + 'NN': 0.4782969000000001, + 'N#': 1.2404672100000003, + 'EI': 0.81, + 'EL': 0.7290000000000001, + 'ES': 0.6561, + 'EE': 0.5904900000000001, + 'EN': 1.341441, + 'E#': 1.2072969000000002, + 'IL': 0.81, + 'IS': 0.7290000000000001, + 'IE': 0.6561, + 'IN': 0.5904900000000001, + 'I#': 0.531441, + 'LS': 0.81, + 'LE': 0.7290000000000001, + 'LN': 0.6561, + 'L#': 0.5904900000000001, + 'SE': 0.81, + 'SN': 0.7290000000000001, + 'S#': 0.6561, + } + ) + for key in gold_counter.keys(): + self.assertAlmostEqual(gold_counter[key], test_counter[key]) + + test_counter = ( QSkipgrams(qval=(4, 6, 5, 1, 0), scaler='SSK') .tokenize('NIALL') - .get_counter(), - Counter( - { - '$$$N': 0.531441, - '$$$I': 0.4782969000000001, - '$$$A': 0.4304672100000001, - '$$$L': 0.7360989291000002, - '$$$#': 0.8504267154039002, - '$$NI': 1.4880348000000003, - '$$NA': 1.3392313200000003, - '$$NL': 2.2900855572000007, - '$$N#': 2.645772003478801, - '$$IA': 1.3392313200000003, - '$$IL': 2.2900855572000007, - '$$I#': 2.645772003478801, - '$$AL': 2.2900855572000007, - '$$A#': 2.645772003478801, - '$$LL': 1.0847773692000002, - '$$L#': 5.291544006957601, - '$$##': 2.460275073345601, - '$NIA': 1.4402051100000002, - '$NIL': 2.462750738100001, - '$NI#': 2.845254813264901, - '$NAL': 2.462750738100001, - '$NA#': 2.845254813264901, - '$NLL': 1.1665661391000004, - '$NL#': 5.690509626529802, - '$N##': 2.645772003478801, - '$IAL': 2.462750738100001, - '$IA#': 2.845254813264901, - '$ILL': 1.1665661391000004, - '$IL#': 5.690509626529802, - '$I##': 2.645772003478801, - '$ALL': 1.1665661391000004, - '$AL#': 5.690509626529802, - '$A##': 2.645772003478801, - '$LL#': 2.845254813264901, - '$L##': 5.291544006957601, - '$###': 0.8504267154039002, - 'NIAL': 1.0097379000000002, - 'NIA#': 1.1665661391000002, - 'NILL': 0.4782969000000001, - 'NIL#': 2.3331322782000004, - 'NI##': 1.0847773692000002, - 'NALL': 0.4782969000000001, - 'NAL#': 2.3331322782000004, - 'NA##': 1.0847773692000002, - 'NLL#': 1.1665661391000002, - 'NL##': 2.1695547384000005, - 'N###': 0.3486784401000001, - 'IALL': 0.531441, - 'IAL#': 2.5923691980000005, - 'IA##': 1.2053081880000003, - 'ILL#': 1.2961845990000003, - 'IL##': 2.4106163760000006, - 'I###': 0.3874204890000001, - 'ALL#': 1.4402051100000004, - 'AL##': 2.6784626400000007, - 'A###': 0.4304672100000001, - 'LL##': 1.4880348000000003, - 'L###': 1.0097379000000002, - '$$$$$N': 0.3486784401000001, - '$$$$$I': 0.31381059609000006, - '$$$$$A': 0.2824295364810001, - '$$$$$L': 0.48295450738251017, - '$$$$$#': 0.8431447750407974, - '$$$$NI': 1.6039208244600003, - '$$$$NA': 1.4435287420140006, - '$$$$NL': 2.468434148843941, - '$$$$N#': 4.309406627986299, - '$$$$IA': 1.4435287420140006, - '$$$$IL': 2.468434148843941, - '$$$$I#': 4.309406627986299, - '$$$$AL': 2.468434148843941, - '$$$$A#': 4.309406627986299, - '$$$$LL': 1.1692582810313406, - '$$$$L#': 8.618813255972597, - '$$$$##': 7.715070145397851, - '$$$NIA': 2.984687447256001, - '$$$NIL': 5.103815534807762, - '$$$NI#': 8.910270709073119, - '$$$NAL': 5.103815534807762, - '$$$NA#': 8.910270709073119, - '$$$NLL': 2.417596832277361, - '$$$NL#': 17.82054141814625, - '$$$N##': 15.951932474542438, - '$$$IAL': 5.103815534807762, - '$$$IA#': 8.910270709073119, - '$$$ILL': 2.417596832277361, - '$$$IL#': 17.82054141814625, - '$$$I##': 15.951932474542438, - '$$$ALL': 2.417596832277361, - '$$$AL#': 17.82054141814625, - '$$$A##': 15.951932474542438, - '$$$LL#': 8.910270709073119, - '$$$L##': 31.903864949084834, - '$$$###': 15.08638445665049, - '$$NIAL': 5.396635688803742, - '$$NIA#': 9.42147782919388, - '$$NILL': 2.556301115749141, - '$$NIL#': 18.84295565838777, - '$$NI##': 16.867139400002937, - '$$NALL': 2.556301115749141, - '$$NAL#': 18.84295565838777, - '$$NA##': 16.867139400002937, - '$$NLL#': 9.42147782919388, - '$$NL##': 33.73427880000585, - '$$N###': 15.951932474542435, - '$$IALL': 2.556301115749141, - '$$IAL#': 18.84295565838777, - '$$IA##': 16.867139400002937, - '$$ILL#': 9.42147782919388, - '$$IL##': 33.73427880000585, - '$$I###': 15.951932474542435, - '$$ALL#': 9.42147782919388, - '$$AL##': 33.73427880000585, - '$$A###': 15.951932474542435, - '$$LL##': 16.867139400002937, - '$$L###': 31.903864949084824, - '$$####': 7.715070145397851, - '$NIALL': 1.4278730800535104, - '$NIAL#': 10.525109490228838, - '$NIA##': 9.421477829193876, - '$NILL#': 5.262554745114417, - '$NIL##': 18.842955658387766, - '$NI###': 8.910270709073117, - '$NALL#': 5.262554745114417, - '$NAL##': 18.842955658387766, - '$NA###': 8.910270709073117, - '$NLL##': 9.421477829193876, - '$NL###': 17.820541418146256, - '$N####': 4.309406627986299, - '$IALL#': 5.262554745114417, - '$IAL##': 18.842955658387766, - '$IA###': 8.910270709073117, - '$ILL##': 9.421477829193876, - '$IL###': 17.820541418146256, - '$I####': 4.309406627986299, - '$ALL##': 9.421477829193876, - '$AL###': 17.820541418146256, - '$A####': 4.309406627986299, - '$LL###': 8.910270709073117, - '$L####': 8.618813255972595, - '$#####': 0.8431447750407974, - 'NIALL#': 1.4278730800535104, - 'NIAL##': 5.112602231498281, - 'NIA###': 2.417596832277361, - 'NILL##': 2.556301115749141, - 'NIL###': 4.835193664554721, - 'NI####': 1.1692582810313406, - 'NALL##': 2.556301115749141, - 'NAL###': 4.835193664554721, - 'NA####': 1.1692582810313406, - 'NLL###': 2.417596832277361, - 'NL####': 2.338516562062681, - 'N#####': 0.2287679245496101, - 'IALL##': 2.8403345730546006, - 'IAL###': 5.3724374050608015, - 'IA####': 1.2991758678126004, - 'ILL###': 2.6862187025304003, - 'IL####': 2.5983517356252004, - 'I#####': 0.2541865828329001, - 'ALL###': 2.984687447256001, - 'AL####': 2.887057484028001, - 'A#####': 0.2824295364810001, - 'LL####': 1.6039208244600003, - 'L#####': 0.6624890361900002, - '$$$$N': 0.4304672100000001, - '$$$$I': 0.3874204890000001, - '$$$$A': 0.3486784401000001, - '$$$$L': 0.5962401325710002, - '$$$$#': 0.8741476583623434, - '$$$NI': 1.5927286770000002, - '$$$NA': 1.4334558093000005, - '$$$NL': 2.4512094339030006, - '$$$N#': 3.59371815104519, - '$$$IA': 1.4334558093000005, - '$$$IL': 2.4512094339030006, - '$$$I#': 3.59371815104519, - '$$$AL': 2.4512094339030006, - '$$$A#': 3.59371815104519, - '$$$LL': 1.1610992055330005, - '$$$L#': 7.187436302090378, - '$$$##': 4.91876456439945, - '$$NIA': 2.2513435083000006, - '$$NIL': 3.849797399193001, - '$$NI#': 5.644187966956859, - '$$NAL': 3.849797399193001, - '$$NA#': 5.644187966956859, - '$$NLL': 1.8235882417230007, - '$$NL#': 11.28837593391372, - '$$N##': 7.725266868411147, - '$$IAL': 3.849797399193001, - '$$IA#': 5.644187966956859, - '$$ILL': 1.8235882417230007, - '$$IL#': 11.28837593391372, - '$$I##': 7.725266868411147, - '$$ALL': 1.8235882417230007, - '$$AL#': 11.28837593391372, - '$$A##': 7.725266868411147, - '$$LL#': 5.644187966956859, - '$$L##': 15.4505337368223, - '$$###': 4.918764564399449, - '$NIAL': 2.812715796861001, - '$NIA#': 4.123722629777913, - '$NILL': 1.3323390616710005, - '$NIL#': 8.247445259555828, - '$NI##': 5.644187966956858, - '$NALL': 1.3323390616710005, - '$NAL#': 8.247445259555828, - '$NA##': 5.644187966956858, - '$NLL#': 4.123722629777913, - '$NL##': 11.288375933913724, - '$N###': 3.593718151045189, - '$IALL': 1.3323390616710005, - '$IAL#': 8.247445259555828, - '$IA##': 5.644187966956858, - '$ILL#': 4.123722629777913, - '$IL##': 11.288375933913724, - '$I###': 3.593718151045189, - '$ALL#': 4.123722629777913, - '$AL##': 11.288375933913724, - '$A###': 3.593718151045189, - '$LL##': 5.644187966956858, - '$L###': 7.187436302090377, - '$####': 0.8741476583623434, - 'NIALL': 0.4304672100000001, - 'NIAL#': 2.664678123342001, - 'NIA##': 1.8235882417230007, - 'NILL#': 1.3323390616710005, - 'NIL##': 3.6471764834460014, - 'NI###': 1.1610992055330005, - 'NALL#': 1.3323390616710005, - 'NAL##': 3.6471764834460014, - 'NA###': 1.1610992055330005, - 'NLL##': 1.8235882417230007, - 'NL###': 2.322198411066001, - 'N####': 0.2824295364810001, - 'IALL#': 1.4803767351900001, - 'IAL##': 4.0524183149400015, - 'IA###': 1.2901102283700003, - 'ILL##': 2.0262091574700007, - 'IL###': 2.5802204567400007, - 'I####': 0.31381059609000006, - 'ALL##': 2.2513435083000006, - 'AL###': 2.8669116186000005, - 'A####': 0.3486784401000001, - 'LL###': 1.5927286770000004, - 'L####': 0.8178876990000001, - 'N': 1.0, - 'I': 1.0, - 'A': 1.0, - 'L': 2.0, - } - ), + .get_counter() + ) + gold_counter = Counter( + { + '$$$N': 0.531441, + '$$$I': 0.4782969000000001, + '$$$A': 0.4304672100000001, + '$$$L': 0.7360989291000002, + '$$$#': 0.8504267154039002, + '$$NI': 1.4880348000000003, + '$$NA': 1.3392313200000003, + '$$NL': 2.2900855572000007, + '$$N#': 2.645772003478801, + '$$IA': 1.3392313200000003, + '$$IL': 2.2900855572000007, + '$$I#': 2.645772003478801, + '$$AL': 2.2900855572000007, + '$$A#': 2.645772003478801, + '$$LL': 1.0847773692000002, + '$$L#': 5.291544006957601, + '$$##': 2.460275073345601, + '$NIA': 1.4402051100000002, + '$NIL': 2.462750738100001, + '$NI#': 2.845254813264901, + '$NAL': 2.462750738100001, + '$NA#': 2.845254813264901, + '$NLL': 1.1665661391000004, + '$NL#': 5.690509626529802, + '$N##': 2.645772003478801, + '$IAL': 2.462750738100001, + '$IA#': 2.845254813264901, + '$ILL': 1.1665661391000004, + '$IL#': 5.690509626529802, + '$I##': 2.645772003478801, + '$ALL': 1.1665661391000004, + '$AL#': 5.690509626529802, + '$A##': 2.645772003478801, + '$LL#': 2.845254813264901, + '$L##': 5.291544006957601, + '$###': 0.8504267154039002, + 'NIAL': 1.0097379000000002, + 'NIA#': 1.1665661391000002, + 'NILL': 0.4782969000000001, + 'NIL#': 2.3331322782000004, + 'NI##': 1.0847773692000002, + 'NALL': 0.4782969000000001, + 'NAL#': 2.3331322782000004, + 'NA##': 1.0847773692000002, + 'NLL#': 1.1665661391000002, + 'NL##': 2.1695547384000005, + 'N###': 0.3486784401000001, + 'IALL': 0.531441, + 'IAL#': 2.5923691980000005, + 'IA##': 1.2053081880000003, + 'ILL#': 1.2961845990000003, + 'IL##': 2.4106163760000006, + 'I###': 0.3874204890000001, + 'ALL#': 1.4402051100000004, + 'AL##': 2.6784626400000007, + 'A###': 0.4304672100000001, + 'LL##': 1.4880348000000003, + 'L###': 1.0097379000000002, + '$$$$$N': 0.3486784401000001, + '$$$$$I': 0.31381059609000006, + '$$$$$A': 0.2824295364810001, + '$$$$$L': 0.48295450738251017, + '$$$$$#': 0.8431447750407974, + '$$$$NI': 1.6039208244600003, + '$$$$NA': 1.4435287420140006, + '$$$$NL': 2.468434148843941, + '$$$$N#': 4.309406627986299, + '$$$$IA': 1.4435287420140006, + '$$$$IL': 2.468434148843941, + '$$$$I#': 4.309406627986299, + '$$$$AL': 2.468434148843941, + '$$$$A#': 4.309406627986299, + '$$$$LL': 1.1692582810313406, + '$$$$L#': 8.618813255972597, + '$$$$##': 7.715070145397851, + '$$$NIA': 2.984687447256001, + '$$$NIL': 5.103815534807762, + '$$$NI#': 8.910270709073119, + '$$$NAL': 5.103815534807762, + '$$$NA#': 8.910270709073119, + '$$$NLL': 2.417596832277361, + '$$$NL#': 17.82054141814625, + '$$$N##': 15.951932474542438, + '$$$IAL': 5.103815534807762, + '$$$IA#': 8.910270709073119, + '$$$ILL': 2.417596832277361, + '$$$IL#': 17.82054141814625, + '$$$I##': 15.951932474542438, + '$$$ALL': 2.417596832277361, + '$$$AL#': 17.82054141814625, + '$$$A##': 15.951932474542438, + '$$$LL#': 8.910270709073119, + '$$$L##': 31.903864949084834, + '$$$###': 15.08638445665049, + '$$NIAL': 5.396635688803742, + '$$NIA#': 9.42147782919388, + '$$NILL': 2.556301115749141, + '$$NIL#': 18.84295565838777, + '$$NI##': 16.867139400002937, + '$$NALL': 2.556301115749141, + '$$NAL#': 18.84295565838777, + '$$NA##': 16.867139400002937, + '$$NLL#': 9.42147782919388, + '$$NL##': 33.73427880000585, + '$$N###': 15.951932474542435, + '$$IALL': 2.556301115749141, + '$$IAL#': 18.84295565838777, + '$$IA##': 16.867139400002937, + '$$ILL#': 9.42147782919388, + '$$IL##': 33.73427880000585, + '$$I###': 15.951932474542435, + '$$ALL#': 9.42147782919388, + '$$AL##': 33.73427880000585, + '$$A###': 15.951932474542435, + '$$LL##': 16.867139400002937, + '$$L###': 31.903864949084824, + '$$####': 7.715070145397851, + '$NIALL': 1.4278730800535104, + '$NIAL#': 10.525109490228838, + '$NIA##': 9.421477829193876, + '$NILL#': 5.262554745114417, + '$NIL##': 18.842955658387766, + '$NI###': 8.910270709073117, + '$NALL#': 5.262554745114417, + '$NAL##': 18.842955658387766, + '$NA###': 8.910270709073117, + '$NLL##': 9.421477829193876, + '$NL###': 17.820541418146256, + '$N####': 4.309406627986299, + '$IALL#': 5.262554745114417, + '$IAL##': 18.842955658387766, + '$IA###': 8.910270709073117, + '$ILL##': 9.421477829193876, + '$IL###': 17.820541418146256, + '$I####': 4.309406627986299, + '$ALL##': 9.421477829193876, + '$AL###': 17.820541418146256, + '$A####': 4.309406627986299, + '$LL###': 8.910270709073117, + '$L####': 8.618813255972595, + '$#####': 0.8431447750407974, + 'NIALL#': 1.4278730800535104, + 'NIAL##': 5.112602231498281, + 'NIA###': 2.417596832277361, + 'NILL##': 2.556301115749141, + 'NIL###': 4.835193664554721, + 'NI####': 1.1692582810313406, + 'NALL##': 2.556301115749141, + 'NAL###': 4.835193664554721, + 'NA####': 1.1692582810313406, + 'NLL###': 2.417596832277361, + 'NL####': 2.338516562062681, + 'N#####': 0.2287679245496101, + 'IALL##': 2.8403345730546006, + 'IAL###': 5.3724374050608015, + 'IA####': 1.2991758678126004, + 'ILL###': 2.6862187025304003, + 'IL####': 2.5983517356252004, + 'I#####': 0.2541865828329001, + 'ALL###': 2.984687447256001, + 'AL####': 2.887057484028001, + 'A#####': 0.2824295364810001, + 'LL####': 1.6039208244600003, + 'L#####': 0.6624890361900002, + '$$$$N': 0.4304672100000001, + '$$$$I': 0.3874204890000001, + '$$$$A': 0.3486784401000001, + '$$$$L': 0.5962401325710002, + '$$$$#': 0.8741476583623434, + '$$$NI': 1.5927286770000002, + '$$$NA': 1.4334558093000005, + '$$$NL': 2.4512094339030006, + '$$$N#': 3.59371815104519, + '$$$IA': 1.4334558093000005, + '$$$IL': 2.4512094339030006, + '$$$I#': 3.59371815104519, + '$$$AL': 2.4512094339030006, + '$$$A#': 3.59371815104519, + '$$$LL': 1.1610992055330005, + '$$$L#': 7.187436302090378, + '$$$##': 4.91876456439945, + '$$NIA': 2.2513435083000006, + '$$NIL': 3.849797399193001, + '$$NI#': 5.644187966956859, + '$$NAL': 3.849797399193001, + '$$NA#': 5.644187966956859, + '$$NLL': 1.8235882417230007, + '$$NL#': 11.28837593391372, + '$$N##': 7.725266868411147, + '$$IAL': 3.849797399193001, + '$$IA#': 5.644187966956859, + '$$ILL': 1.8235882417230007, + '$$IL#': 11.28837593391372, + '$$I##': 7.725266868411147, + '$$ALL': 1.8235882417230007, + '$$AL#': 11.28837593391372, + '$$A##': 7.725266868411147, + '$$LL#': 5.644187966956859, + '$$L##': 15.4505337368223, + '$$###': 4.918764564399449, + '$NIAL': 2.812715796861001, + '$NIA#': 4.123722629777913, + '$NILL': 1.3323390616710005, + '$NIL#': 8.247445259555828, + '$NI##': 5.644187966956858, + '$NALL': 1.3323390616710005, + '$NAL#': 8.247445259555828, + '$NA##': 5.644187966956858, + '$NLL#': 4.123722629777913, + '$NL##': 11.288375933913724, + '$N###': 3.593718151045189, + '$IALL': 1.3323390616710005, + '$IAL#': 8.247445259555828, + '$IA##': 5.644187966956858, + '$ILL#': 4.123722629777913, + '$IL##': 11.288375933913724, + '$I###': 3.593718151045189, + '$ALL#': 4.123722629777913, + '$AL##': 11.288375933913724, + '$A###': 3.593718151045189, + '$LL##': 5.644187966956858, + '$L###': 7.187436302090377, + '$####': 0.8741476583623434, + 'NIALL': 0.4304672100000001, + 'NIAL#': 2.664678123342001, + 'NIA##': 1.8235882417230007, + 'NILL#': 1.3323390616710005, + 'NIL##': 3.6471764834460014, + 'NI###': 1.1610992055330005, + 'NALL#': 1.3323390616710005, + 'NAL##': 3.6471764834460014, + 'NA###': 1.1610992055330005, + 'NLL##': 1.8235882417230007, + 'NL###': 2.322198411066001, + 'N####': 0.2824295364810001, + 'IALL#': 1.4803767351900001, + 'IAL##': 4.0524183149400015, + 'IA###': 1.2901102283700003, + 'ILL##': 2.0262091574700007, + 'IL###': 2.5802204567400007, + 'I####': 0.31381059609000006, + 'ALL##': 2.2513435083000006, + 'AL###': 2.8669116186000005, + 'A####': 0.3486784401000001, + 'LL###': 1.5927286770000004, + 'L####': 0.8178876990000001, + 'N': 1.0, + 'I': 1.0, + 'A': 1.0, + 'L': 2.0, + } ) + for key in gold_counter.keys(): + self.assertAlmostEqual(gold_counter[key], test_counter[key]) + self.assertEqual( QSkipgrams(qval=(2, 3), scaler='length') .tokenize('NIALL') @@ -492,126 +499,131 @@ def test_qskipgrams(self): } ), ) - self.assertEqual( + test_counter = ( QSkipgrams(qval=(2, 3), scaler='length-log') .tokenize('NIALL') - .get_counter(), - Counter( - { - '$N': 1.0986122886681096, - '$I': 1.0986122886681096, - '$A': 1.0986122886681096, - '$L': 2.197224577336219, - '$#': 1.0986122886681096, - 'NI': 1.0986122886681096, - 'NA': 1.0986122886681096, - 'NL': 2.197224577336219, - 'N#': 1.0986122886681096, - 'IA': 1.0986122886681096, - 'IL': 2.197224577336219, - 'I#': 1.0986122886681096, - 'AL': 2.197224577336219, - 'A#': 1.0986122886681096, - 'LL': 1.0986122886681096, - 'L#': 2.197224577336219, - '$$N': 1.3862943611198906, - '$$I': 1.3862943611198906, - '$$A': 1.3862943611198906, - '$$L': 2.772588722239781, - '$$#': 2.772588722239781, - '$NI': 2.772588722239781, - '$NA': 2.772588722239781, - '$NL': 5.545177444479562, - '$N#': 5.545177444479562, - '$IA': 2.772588722239781, - '$IL': 5.545177444479562, - '$I#': 5.545177444479562, - '$AL': 5.545177444479562, - '$A#': 5.545177444479562, - '$LL': 2.772588722239781, - '$L#': 11.090354888959125, - '$##': 2.772588722239781, - 'NIA': 1.3862943611198906, - 'NIL': 2.772588722239781, - 'NI#': 2.772588722239781, - 'NAL': 2.772588722239781, - 'NA#': 2.772588722239781, - 'NLL': 1.3862943611198906, - 'NL#': 5.545177444479562, - 'N##': 1.3862943611198906, - 'IAL': 2.772588722239781, - 'IA#': 2.772588722239781, - 'ILL': 1.3862943611198906, - 'IL#': 5.545177444479562, - 'I##': 1.3862943611198906, - 'ALL': 1.3862943611198906, - 'AL#': 5.545177444479562, - 'A##': 1.3862943611198906, - 'LL#': 2.772588722239781, - 'L##': 2.772588722239781, - } - ), + .get_counter() ) - self.assertEqual( + gold_counter = Counter( + { + '$N': 1.0986122886681096, + '$I': 1.0986122886681096, + '$A': 1.0986122886681096, + '$L': 2.197224577336219, + '$#': 1.0986122886681096, + 'NI': 1.0986122886681096, + 'NA': 1.0986122886681096, + 'NL': 2.197224577336219, + 'N#': 1.0986122886681096, + 'IA': 1.0986122886681096, + 'IL': 2.197224577336219, + 'I#': 1.0986122886681096, + 'AL': 2.197224577336219, + 'A#': 1.0986122886681096, + 'LL': 1.0986122886681096, + 'L#': 2.197224577336219, + '$$N': 1.3862943611198906, + '$$I': 1.3862943611198906, + '$$A': 1.3862943611198906, + '$$L': 2.772588722239781, + '$$#': 2.772588722239781, + '$NI': 2.772588722239781, + '$NA': 2.772588722239781, + '$NL': 5.545177444479562, + '$N#': 5.545177444479562, + '$IA': 2.772588722239781, + '$IL': 5.545177444479562, + '$I#': 5.545177444479562, + '$AL': 5.545177444479562, + '$A#': 5.545177444479562, + '$LL': 2.772588722239781, + '$L#': 11.090354888959125, + '$##': 2.772588722239781, + 'NIA': 1.3862943611198906, + 'NIL': 2.772588722239781, + 'NI#': 2.772588722239781, + 'NAL': 2.772588722239781, + 'NA#': 2.772588722239781, + 'NLL': 1.3862943611198906, + 'NL#': 5.545177444479562, + 'N##': 1.3862943611198906, + 'IAL': 2.772588722239781, + 'IA#': 2.772588722239781, + 'ILL': 1.3862943611198906, + 'IL#': 5.545177444479562, + 'I##': 1.3862943611198906, + 'ALL': 1.3862943611198906, + 'AL#': 5.545177444479562, + 'A##': 1.3862943611198906, + 'LL#': 2.772588722239781, + 'L##': 2.772588722239781, + } + ) + for key in gold_counter.keys(): + self.assertAlmostEqual(gold_counter[key], test_counter[key]) + + test_counter = ( QSkipgrams(qval=(2, 3), scaler='length-exp') .tokenize('NIALL') - .get_counter(), - Counter( - { - '$N': 7.38905609893065, - '$I': 7.38905609893065, - '$A': 7.38905609893065, - '$L': 14.7781121978613, - '$#': 7.38905609893065, - 'NI': 7.38905609893065, - 'NA': 7.38905609893065, - 'NL': 14.7781121978613, - 'N#': 7.38905609893065, - 'IA': 7.38905609893065, - 'IL': 14.7781121978613, - 'I#': 7.38905609893065, - 'AL': 14.7781121978613, - 'A#': 7.38905609893065, - 'LL': 7.38905609893065, - 'L#': 14.7781121978613, - '$$N': 20.085536923187668, - '$$I': 20.085536923187668, - '$$A': 20.085536923187668, - '$$L': 40.171073846375336, - '$$#': 40.171073846375336, - '$NI': 40.171073846375336, - '$NA': 40.171073846375336, - '$NL': 80.34214769275067, - '$N#': 80.34214769275067, - '$IA': 40.171073846375336, - '$IL': 80.34214769275067, - '$I#': 80.34214769275067, - '$AL': 80.34214769275067, - '$A#': 80.34214769275067, - '$LL': 40.171073846375336, - '$L#': 160.68429538550137, - '$##': 40.171073846375336, - 'NIA': 20.085536923187668, - 'NIL': 40.171073846375336, - 'NI#': 40.171073846375336, - 'NAL': 40.171073846375336, - 'NA#': 40.171073846375336, - 'NLL': 20.085536923187668, - 'NL#': 80.34214769275067, - 'N##': 20.085536923187668, - 'IAL': 40.171073846375336, - 'IA#': 40.171073846375336, - 'ILL': 20.085536923187668, - 'IL#': 80.34214769275067, - 'I##': 20.085536923187668, - 'ALL': 20.085536923187668, - 'AL#': 80.34214769275067, - 'A##': 20.085536923187668, - 'LL#': 40.171073846375336, - 'L##': 40.171073846375336, - } - ), + .get_counter() + ) + gold_counter = Counter( + { + '$N': 7.38905609893065, + '$I': 7.38905609893065, + '$A': 7.38905609893065, + '$L': 14.7781121978613, + '$#': 7.38905609893065, + 'NI': 7.38905609893065, + 'NA': 7.38905609893065, + 'NL': 14.7781121978613, + 'N#': 7.38905609893065, + 'IA': 7.38905609893065, + 'IL': 14.7781121978613, + 'I#': 7.38905609893065, + 'AL': 14.7781121978613, + 'A#': 7.38905609893065, + 'LL': 7.38905609893065, + 'L#': 14.7781121978613, + '$$N': 20.085536923187668, + '$$I': 20.085536923187668, + '$$A': 20.085536923187668, + '$$L': 40.171073846375336, + '$$#': 40.171073846375336, + '$NI': 40.171073846375336, + '$NA': 40.171073846375336, + '$NL': 80.34214769275067, + '$N#': 80.34214769275067, + '$IA': 40.171073846375336, + '$IL': 80.34214769275067, + '$I#': 80.34214769275067, + '$AL': 80.34214769275067, + '$A#': 80.34214769275067, + '$LL': 40.171073846375336, + '$L#': 160.68429538550137, + '$##': 40.171073846375336, + 'NIA': 20.085536923187668, + 'NIL': 40.171073846375336, + 'NI#': 40.171073846375336, + 'NAL': 40.171073846375336, + 'NA#': 40.171073846375336, + 'NLL': 20.085536923187668, + 'NL#': 80.34214769275067, + 'N##': 20.085536923187668, + 'IAL': 40.171073846375336, + 'IA#': 40.171073846375336, + 'ILL': 20.085536923187668, + 'IL#': 80.34214769275067, + 'I##': 20.085536923187668, + 'ALL': 20.085536923187668, + 'AL#': 80.34214769275067, + 'A##': 20.085536923187668, + 'LL#': 40.171073846375336, + 'L##': 40.171073846375336, + } ) + for key in gold_counter.keys(): + self.assertAlmostEqual(gold_counter[key], test_counter[key]) if __name__ == '__main__': From 06e769d292971f9d6e2bb56eececeb6e7ecd7225 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 15:23:53 -0800 Subject: [PATCH 13/35] added win1803 (container windows) target & bumped maxParallel to 10 --- azure-pipelines.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7f12f0e1a..fc2278260 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -54,7 +54,22 @@ jobs: Python38MacOS: python.version: '3.8' imageName: 'macOS-latest' - maxParallel: 4 + Python27Win1803: + python.version: '2.7' + imageName: 'win1803' + Python35Win1803: + python.version: '3.5' + imageName: 'win1803' + Python36Win1803: + python.version: '3.6' + imageName: 'win1803' + Python37Win1803: + python.version: '3.7' + imageName: 'win1803' + Python38Win1803: + python.version: '3.8' + imageName: 'win1803' + maxParallel: 10 pool: vmImage: $(imageName) From 7631e75186a5a5dfc6b1568cc82d1b14c600cfa4 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 15:32:37 -0800 Subject: [PATCH 14/35] added fix for py36 regression tests --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 8c6b85c8e..8440a2c43 100644 --- a/tox.ini +++ b/tox.ini @@ -46,8 +46,8 @@ commands = nosetests --where={toxinidir}/abydos [] deps = nose deprecation - py37: lzss - py37: paq + py37, py36: lzss + py37, py36: paq commands = nosetests {toxinidir}/tests/regression --processes=-1 \ --process-timeout=1200 --process-restartworker -c=0 -v [] From efdae384387d75be753558ac87cf3c126519d3cf Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 15:41:25 -0800 Subject: [PATCH 15/35] removed win1803 (doesn't have Python installed) --- azure-pipelines.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fc2278260..d0275cb41 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -54,21 +54,6 @@ jobs: Python38MacOS: python.version: '3.8' imageName: 'macOS-latest' - Python27Win1803: - python.version: '2.7' - imageName: 'win1803' - Python35Win1803: - python.version: '3.5' - imageName: 'win1803' - Python36Win1803: - python.version: '3.6' - imageName: 'win1803' - Python37Win1803: - python.version: '3.7' - imageName: 'win1803' - Python38Win1803: - python.version: '3.8' - imageName: 'win1803' maxParallel: 10 pool: From 2b713819a4150d7edf5ea265cbce8a40009c7b40 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 15:42:18 -0800 Subject: [PATCH 16/35] fixed target formatting & added py36-doctest for semaphore --- tox.ini | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 8440a2c43..995528f5e 100644 --- a/tox.ini +++ b/tox.ini @@ -42,12 +42,18 @@ passenv = NOSE_WITH_DOCTEST commands = nosetests --where={toxinidir}/abydos [] +[testenv:py36-doctest] +basepython = python3.6 +setenv = {[testenv:doctest]setenv} +passenv = {[testenv:doctest]passenv} +commands = {[testenv:doctest]commands} + [testenv:regression] deps = nose deprecation - py37, py36: lzss - py37, py36: paq + py37,py36: lzss + py37,py36: paq commands = nosetests {toxinidir}/tests/regression --processes=-1 \ --process-timeout=1200 --process-restartworker -c=0 -v [] @@ -68,8 +74,8 @@ deps = nose nltk syllabipy - py37, py36: lzss - py37, py36: paq + py37,py36: lzss + py37,py36: paq commands = nosetests {toxinidir}/tests/fuzz --processes=-1 \ --process-timeout=1200 --process-restartworker -c=0 -v [] From 2ad87b2b3ca21e76e74a22aed9032e0b2111e04f Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 15:46:25 -0800 Subject: [PATCH 17/35] added x86 pythons for windows --- azure-pipelines.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d0275cb41..ce0eebfa7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -39,6 +39,26 @@ jobs: Python38Windows: python.version: '3.8' imageName: 'windows-latest' + Python27Windowsx86: + python.version: '2.7' + imageName: 'windows-latest' + architecture: 'x86' + Python35Windowsx86: + python.version: '3.5' + imageName: 'windows-latest' + architecture: 'x86' + Python36Windowsx86: + python.version: '3.6' + imageName: 'windows-latest' + architecture: 'x86' + Python37Windowsx86: + python.version: '3.7' + imageName: 'windows-latest' + architecture: 'x86' + Python38Windowsx86: + python.version: '3.8' + imageName: 'windows-latest' + architecture: 'x86' Python27MacOS: python.version: '2.7' imageName: 'macOS-latest' @@ -63,7 +83,7 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '$(python.version)' - architecture: 'x64' + architecture: '$(architecture)' - script: python -m pip install --upgrade pip && pip install -r requirements.txt && pip install -r requirements-test.txt displayName: 'Install dependencies' From 085387bb34478104654a7f7b69f22a753453ebd5 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Mon, 6 Jan 2020 16:00:17 -0800 Subject: [PATCH 18/35] added x64 as architecture for remaining platforms --- azure-pipelines.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ce0eebfa7..7299b6d58 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,33 +12,43 @@ jobs: Python27Linux: python.version: '2.7' imageName: 'ubuntu-latest' + architecture: 'x64' Python35Linux: python.version: '3.5' imageName: 'ubuntu-latest' + architecture: 'x64' Python36Linux: python.version: '3.6' imageName: 'ubuntu-latest' + architecture: 'x64' Python37Linux: python.version: '3.7' imageName: 'ubuntu-latest' + architecture: 'x64' Python38Linux: python.version: '3.8' imageName: 'ubuntu-latest' + architecture: 'x64' Python27Windows: python.version: '2.7' imageName: 'windows-latest' + architecture: 'x64' Python35Windows: python.version: '3.5' imageName: 'windows-latest' + architecture: 'x64' Python36Windows: python.version: '3.6' imageName: 'windows-latest' + architecture: 'x64' Python37Windows: python.version: '3.7' imageName: 'windows-latest' + architecture: 'x64' Python38Windows: python.version: '3.8' imageName: 'windows-latest' + architecture: 'x64' Python27Windowsx86: python.version: '2.7' imageName: 'windows-latest' @@ -62,18 +72,23 @@ jobs: Python27MacOS: python.version: '2.7' imageName: 'macOS-latest' + architecture: 'x64' Python35MacOS: python.version: '3.5' imageName: 'macOS-latest' + architecture: 'x64' Python36MacOS: python.version: '3.6' imageName: 'macOS-latest' + architecture: 'x64' Python37MacOS: python.version: '3.7' imageName: 'macOS-latest' + architecture: 'x64' Python38MacOS: python.version: '3.8' imageName: 'macOS-latest' + architecture: 'x64' maxParallel: 10 pool: From 582489038539ca38004f2bf981e1a8dd3f0b0b2b Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 11:04:39 -0800 Subject: [PATCH 19/35] added temporary fix for coverage bug #916 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 995528f5e..1803b4d76 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,7 @@ indexserver = deps = :preinstall: cython nose - coverage + coverage <5.0.2 py27: pyliblzma lzss py37,py36,doctest: paq From dcbc5d4fd9175401b2086d5e3380f9806a2b6c06 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 11:08:24 -0800 Subject: [PATCH 20/35] simplified to != for future fix --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1803b4d76..ff05fb6f4 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,7 @@ indexserver = deps = :preinstall: cython nose - coverage <5.0.2 + coverage !=5.0.2 py27: pyliblzma lzss py37,py36,doctest: paq From 0580b14bbe3408534162b1b342153fab5e3779ee Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 11:08:43 -0800 Subject: [PATCH 21/35] added nightly & simplified case to if --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a6cee0acb..e2bd9b34c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - 3.6 - 3.7 - 3.8 + - nightly notifications: email: false @@ -14,8 +15,8 @@ notifications: # Install packages install: - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install pyliblzma; fi - - case "$TRAVIS_PYTHON_VERSION" in 3.5|3.6|3.7|3.8) travis_retry pip install cython;; esac - - case "$TRAVIS_PYTHON_VERSION" in 3.5|3.6|3.7|3.8) travis_retry pip install paq lzss;; esac + - if [[ $TRAVIS_PYTHON_VERSION != 2* ]]; travis_retry pip install cython;; fi + - if [[ $TRAVIS_PYTHON_VERSION != 2* ]]; travis_retry pip install paq lzss;; fi - travis_retry pip install coveralls - travis_retry pip install -r requirements.txt - travis_retry pip install -r requirements-test.txt From 01aa32c3316cf2ffe1f5b095905977e435dc727b Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 11:16:16 -0800 Subject: [PATCH 22/35] adjusted logic & added new oses & architectures --- .travis.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e2bd9b34c..665d77d5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,14 +9,27 @@ python: - 3.8 - nightly +jobs: + include: + - os: osx + - os: windows + - os: linux + arch: amd64 + - os: linux + arch: ppc64le + - os: linux + arch: s390x + - os: linux + arch: arm64 + notifications: email: false # Install packages install: - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install pyliblzma; fi - - if [[ $TRAVIS_PYTHON_VERSION != 2* ]]; travis_retry pip install cython;; fi - - if [[ $TRAVIS_PYTHON_VERSION != 2* ]]; travis_retry pip install paq lzss;; fi + - if ![[ $TRAVIS_PYTHON_VERSION == 2* ]]; travis_retry pip install cython;; fi + - if ![[ $TRAVIS_PYTHON_VERSION == 2* ]]; travis_retry pip install paq lzss;; fi - travis_retry pip install coveralls - travis_retry pip install -r requirements.txt - travis_retry pip install -r requirements-test.txt From d1f9ce02e56f5a60a5c54f8aaf9e0e7c7993b661 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 11:18:05 -0800 Subject: [PATCH 23/35] added then --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 665d77d5d..ddd0e7447 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,8 +28,8 @@ notifications: # Install packages install: - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install pyliblzma; fi - - if ![[ $TRAVIS_PYTHON_VERSION == 2* ]]; travis_retry pip install cython;; fi - - if ![[ $TRAVIS_PYTHON_VERSION == 2* ]]; travis_retry pip install paq lzss;; fi + - if ![[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install cython;; fi + - if ![[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install paq lzss;; fi - travis_retry pip install coveralls - travis_retry pip install -r requirements.txt - travis_retry pip install -r requirements-test.txt From 6e9a889e02bde0b9cf0b49e643f6fe841be08cfe Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 11:20:10 -0800 Subject: [PATCH 24/35] switched back to != --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ddd0e7447..72c000af3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ python: jobs: include: - os: osx - - os: windows - os: linux arch: amd64 - os: linux @@ -28,8 +27,8 @@ notifications: # Install packages install: - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install pyliblzma; fi - - if ![[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install cython;; fi - - if ![[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install paq lzss;; fi + - if [[ $TRAVIS_PYTHON_VERSION != 2* ]]; then travis_retry pip install cython;; fi + - if [[ $TRAVIS_PYTHON_VERSION != 2* ]]; then travis_retry pip install paq lzss;; fi - travis_retry pip install coveralls - travis_retry pip install -r requirements.txt - travis_retry pip install -r requirements-test.txt From a759789e193c44971f61fee8b53159d2c184fcee Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 11:24:42 -0800 Subject: [PATCH 25/35] default to 3.7 for non-xenial; swtiched to == 3* --- .travis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 72c000af3..d07dee26d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,14 +12,16 @@ python: jobs: include: - os: osx - - os: linux - arch: amd64 + python: 3.7 - os: linux arch: ppc64le + python: 3.7 - os: linux arch: s390x + python: 3.7 - os: linux arch: arm64 + python: 3.7 notifications: email: false @@ -27,8 +29,8 @@ notifications: # Install packages install: - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install pyliblzma; fi - - if [[ $TRAVIS_PYTHON_VERSION != 2* ]]; then travis_retry pip install cython;; fi - - if [[ $TRAVIS_PYTHON_VERSION != 2* ]]; then travis_retry pip install paq lzss;; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then travis_retry pip install cython;; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then travis_retry pip install paq lzss;; fi - travis_retry pip install coveralls - travis_retry pip install -r requirements.txt - travis_retry pip install -r requirements-test.txt From 454379cd1d60a0f6aedc8dae90c8c0a8f5f85b70 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 11:26:56 -0800 Subject: [PATCH 26/35] remove extra ; --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d07dee26d..fd6f0b1db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,8 +29,8 @@ notifications: # Install packages install: - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install pyliblzma; fi - - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then travis_retry pip install cython;; fi - - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then travis_retry pip install paq lzss;; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then travis_retry pip install cython; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then travis_retry pip install paq lzss; fi - travis_retry pip install coveralls - travis_retry pip install -r requirements.txt - travis_retry pip install -r requirements-test.txt From 487fd13dbcd93cd8750637413d47d3941c2617f5 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 11:47:07 -0800 Subject: [PATCH 27/35] put alternate oses/arches at top to run earlier --- .travis.yml | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd6f0b1db..812ab6203 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,16 @@ language: python +jobs: + include: + - os: linux + arch: arm64 + python: 3.7 + - os: linux + arch: ppc64le + python: 3.7 + - os: osx + python: 3.6 + dist: xenial python: - 2.7 @@ -9,28 +20,14 @@ python: - 3.8 - nightly -jobs: - include: - - os: osx - python: 3.7 - - os: linux - arch: ppc64le - python: 3.7 - - os: linux - arch: s390x - python: 3.7 - - os: linux - arch: arm64 - python: 3.7 - notifications: email: false # Install packages install: - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install pyliblzma; fi - - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then travis_retry pip install cython; fi - - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then travis_retry pip install paq lzss; fi + - if [[ $TRAVIS_PYTHON_VERSION != 2* ]]; then travis_retry pip install cython; fi + - if [[ $TRAVIS_PYTHON_VERSION != 2* ]]; then travis_retry pip install paq lzss; fi - travis_retry pip install coveralls - travis_retry pip install -r requirements.txt - travis_retry pip install -r requirements-test.txt From bf85233466d5d449ef7768e9a11d6e2c92ec9257 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 11:49:28 -0800 Subject: [PATCH 28/35] embedded xenial builds in jobs --- .travis.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 812ab6203..0ceb5dab3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,15 +10,16 @@ jobs: python: 3.7 - os: osx python: 3.6 - -dist: xenial -python: - - 2.7 - - 3.5 - - 3.6 - - 3.7 - - 3.8 - - nightly + - os: linux + arch: amd64 + dist: xenial + python: + - 2.7 + - 3.5 + - 3.6 + - 3.7 + - 3.8 + - nightly notifications: email: false From da068fb7100d9dd70f6078a80e0d7de5ab19efe7 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 11:50:36 -0800 Subject: [PATCH 29/35] undid embedding --- .travis.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0ceb5dab3..812ab6203 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,16 +10,15 @@ jobs: python: 3.7 - os: osx python: 3.6 - - os: linux - arch: amd64 - dist: xenial - python: - - 2.7 - - 3.5 - - 3.6 - - 3.7 - - 3.8 - - nightly + +dist: xenial +python: + - 2.7 + - 3.5 + - 3.6 + - 3.7 + - 3.8 + - nightly notifications: email: false From f70fde9e6ea491afadae6d1f03bf900973128bed Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 11:55:16 -0800 Subject: [PATCH 30/35] disable nightly test --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 812ab6203..b044c149b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,6 @@ python: - 3.6 - 3.7 - 3.8 - - nightly notifications: email: false From 2bb6148d9ac13450259fdf87f05796548aeeb869 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 12:05:23 -0800 Subject: [PATCH 31/35] updated DOI & release date --- HISTORY.rst | 6 ++---- README.rst | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fe050e849..231554baf 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,12 +1,10 @@ Release History --------------- -0.4.1 (2019-07-00) *distant dietrich* +0.4.1 (2020-01-07) *distant dietrich* +++++++++++++++++++++++++++++++++++++ -doi: - -Version 0.4.1 ... +doi:10.5281/zenodo.3600548 Changes: diff --git a/README.rst b/README.rst index 159aa3838..717ada1c5 100644 --- a/README.rst +++ b/README.rst @@ -107,8 +107,8 @@ Abydos :target: https://libraries.io/pypi/abydos :alt: Libraries.io SourceRank -.. |zenodo| image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3235034.svg - :target: https://doi.org/10.5281/zenodo.3235034 +.. |zenodo| image:: https://zenodo.org/badge/DOI/10.5281/zenodo.3600548 + :target: https://doi.org/10.5281/zenodo.3600548 :alt: Zenodo .. |cii| image:: https://bestpractices.coreinfrastructure.org/projects/1598/badge From 08feb233084176a2342edf721f66e135ebd973e8 Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 12:30:59 -0800 Subject: [PATCH 32/35] updated README for new release --- README.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 717ada1c5..159b8a049 100644 --- a/README.rst +++ b/README.rst @@ -71,11 +71,11 @@ Abydos :target: https://app.fossa.io/projects/git%2Bgithub.com%2Fchrislit%2Fabydos?ref=badge_shield :alt: FOSSA Status -.. |pylint| image:: https://img.shields.io/badge/Pylint-8.87/10-yellowgreen.svg +.. |pylint| image:: https://img.shields.io/badge/Pylint-9.0/10-yellowgreen.svg :target: # :alt: Pylint Score -.. |flake8| image:: https://img.shields.io/badge/flake8-1-green.svg +.. |flake8| image:: https://img.shields.io/badge/flake8-0-brightgreen.svg :target: # :alt: flake8 Errors @@ -83,7 +83,7 @@ Abydos :target: # :alt: pydocstyle Errors -.. |sloccount| image:: https://img.shields.io/badge/SLOCCount-42,187-blue.svg +.. |sloccount| image:: https://img.shields.io/badge/SLOCCount-43,113-blue.svg :target: # :alt: SLOCCount @@ -164,7 +164,7 @@ Abydos | | `Abydos NLP/IR library `_ -| Copyright 2014-2019 by Christopher C. Little +| Copyright 2014-2020 by Christopher C. Little Abydos is a library of phonetic algorithms, string distance measures & metrics, stemmers, and string fingerprinters including: @@ -315,7 +315,7 @@ To install from `conda-forge `_:: conda install abydos -It should run on Python 2.7 and Python 3.5-3.7. +It should run on Python 2.7 and Python 3.5-3.8. Testing & Contributing ====================== @@ -324,10 +324,10 @@ To run the whole test-suite just call tox:: tox -The tox setup has the following environments: black, py36, py27, doctest, -py36-regression, py27-regression, py36-fuzz, py27-fuzz, pylint, pycodestyle, -pydocstyle, flake8, doc8, badges, docs, & dist. So if you only want to generate -documentation (in HTML, EPUB, & PDF formats), just call:: +The tox setup has the following environments: black, py37, py27, doctest, +py37-regression, py27-regression, py37-fuzz, py27-fuzz, pylint, pydocstyle, +flake8, doc8, docs, sloccount, badges, & build. So if you only want to +generate documentation (in HTML, EPUB, & PDF formats), just call:: tox -e docs From e5854c8402b7eb4605b5ec3d835c2dddd905d1bf Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 12:34:44 -0800 Subject: [PATCH 33/35] updated copyright --- abydos/distance/_token_distance.py | 2 +- abydos/fingerprint/_extract.py | 2 +- abydos/phones/__init__.py | 2 +- abydos/phones/_phones.py | 2 +- tests/distance/test_distance_chao_dice.py | 2 +- tests/distance/test_distance_chao_jaccard.py | 2 +- tests/phones/test_phones.py | 2 +- tests/tokenizer/test_tokenizer_q_skipgrams.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/abydos/distance/_token_distance.py b/abydos/distance/_token_distance.py index b05ab76fe..1eb8db4ee 100644 --- a/abydos/distance/_token_distance.py +++ b/abydos/distance/_token_distance.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2018-2019 by Christopher C. Little. +# Copyright 2018-2020 by Christopher C. Little. # This file is part of Abydos. # # Abydos is free software: you can redistribute it and/or modify diff --git a/abydos/fingerprint/_extract.py b/abydos/fingerprint/_extract.py index 936851fc0..aa6787e7f 100644 --- a/abydos/fingerprint/_extract.py +++ b/abydos/fingerprint/_extract.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 by Christopher C. Little. +# Copyright 2019-2020 by Christopher C. Little. # This file is part of Abydos. # # Abydos is free software: you can redistribute it and/or modify diff --git a/abydos/phones/__init__.py b/abydos/phones/__init__.py index ea7ab6110..b2b0da057 100644 --- a/abydos/phones/__init__.py +++ b/abydos/phones/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2014-2018 by Christopher C. Little. +# Copyright 2014-2020 by Christopher C. Little. # This file is part of Abydos. # # Abydos is free software: you can redistribute it and/or modify diff --git a/abydos/phones/_phones.py b/abydos/phones/_phones.py index 1372d0cb4..16be46f10 100644 --- a/abydos/phones/_phones.py +++ b/abydos/phones/_phones.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2014-2019 by Christopher C. Little. +# Copyright 2014-2020 by Christopher C. Little. # This file is part of Abydos. # # Abydos is free software: you can redistribute it and/or modify diff --git a/tests/distance/test_distance_chao_dice.py b/tests/distance/test_distance_chao_dice.py index 683d2de6d..13b7b8bca 100644 --- a/tests/distance/test_distance_chao_dice.py +++ b/tests/distance/test_distance_chao_dice.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 by Christopher C. Little. +# Copyright 2019-2020 by Christopher C. Little. # This file is part of Abydos. # # Abydos is free software: you can redistribute it and/or modify diff --git a/tests/distance/test_distance_chao_jaccard.py b/tests/distance/test_distance_chao_jaccard.py index 6fd39aa98..4379d6bc0 100644 --- a/tests/distance/test_distance_chao_jaccard.py +++ b/tests/distance/test_distance_chao_jaccard.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 by Christopher C. Little. +# Copyright 2019-2020 by Christopher C. Little. # This file is part of Abydos. # # Abydos is free software: you can redistribute it and/or modify diff --git a/tests/phones/test_phones.py b/tests/phones/test_phones.py index 20cfd81d0..e9e5772b0 100644 --- a/tests/phones/test_phones.py +++ b/tests/phones/test_phones.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2014-2018 by Christopher C. Little. +# Copyright 2014-2020 by Christopher C. Little. # This file is part of Abydos. # # Abydos is free software: you can redistribute it and/or modify diff --git a/tests/tokenizer/test_tokenizer_q_skipgrams.py b/tests/tokenizer/test_tokenizer_q_skipgrams.py index 54518b9f0..dcbcbb432 100644 --- a/tests/tokenizer/test_tokenizer_q_skipgrams.py +++ b/tests/tokenizer/test_tokenizer_q_skipgrams.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2019 by Christopher C. Little. +# Copyright 2019-2020 by Christopher C. Little. # This file is part of Abydos. # # Abydos is free software: you can redistribute it and/or modify From f5b7c0ce5b2154931448d147361cbb6e4bf3014a Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 13:06:52 -0800 Subject: [PATCH 34/35] force a more recent image to get py3.6 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b044c149b..93066589e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ jobs: arch: ppc64le python: 3.7 - os: osx + osx_image: xcode11 python: 3.6 dist: xenial From fa6c88c8e16a59d50fd493077d6d4bc953c6e31f Mon Sep 17 00:00:00 2001 From: Chris Little Date: Tue, 7 Jan 2020 13:09:06 -0800 Subject: [PATCH 35/35] fix to install Cython early on Semaphore --- tox.ini | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index ff05fb6f4..73e75d6fa 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,9 @@ deps = {[testenv:regression]deps} commands = {[testenv:regression]commands} [testenv:py36-regression] -deps = {[testenv:regression]deps} +deps = + :preinstall: cython + {[testenv:regression]deps} commands = {[testenv:regression]commands} [testenv:fuzz] @@ -88,7 +90,9 @@ deps = {[testenv:fuzz]deps} commands = {[testenv:fuzz]commands} [testenv:py36-fuzz] -deps = {[testenv:fuzz]deps} +deps = + :preinstall: cython + {[testenv:fuzz]deps} commands = {[testenv:fuzz]commands} [testenv:pylint]