Changing order of execution in CategoricalEvaluator (#5)

* adding appveyor and changing the order of execution for the categorical eval

* Bump version: 0.0.3 → 0.0.4

* for some reason i thought python3.7 was a thing. removing

* whoops did not want to run ci scripts on develop

* removing pypy because that continues to befuddle me

* removing pypy from appveyor

.travis.yml

@@ -1,3 +1,6 @@
+branches:
+  only:
+  - master
 sudo: false
 language: python
 matrix:
@@ -13,7 +16,28 @@ matrix:
   - python: 3.5
     env: TOXENV=flake8
 install:
-- pip install tox
+  - pip install tox
+  - |
+    set -ex
+    if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then
+        (cd $HOME
+         wget https://bitbucket.org/pypy/pypy/downloads/pypy2-v6.0.0-linux64.tar.bz2
+         tar xf pypy2-*.tar.bz2
+         pypy2-*/bin/pypy -m ensurepip
+         pypy2-*/bin/pypy -m pip install -U virtualenv)
+        export PATH=$(echo $HOME/pypy2-*/bin):$PATH
+        export TOXPYTHON=$(echo $HOME/pypy2-*/bin/pypy)
+    fi
+    if [[ $TRAVIS_PYTHON_VERSION == 'pypy3' ]]; then
+        (cd $HOME
+         wget https://bitbucket.org/pypy/pypy/downloads/pypy3-v6.0.0-linux64.tar.bz2
+         tar xf pypy3-*.tar.bz2
+         pypy3-*/bin/pypy3 -m ensurepip
+         pypy3-*/bin/pypy3 -m pip install -U virtualenv)
+        export PATH=$(echo $HOME/pypy3-*/bin):$PATH
+        export TOXPYTHON=$(echo $HOME/pypy3-*/bin/pypy3)
+    fi
+    set +x
 script: tox
 after_success:
 - pip install codecov
@@ -22,6 +46,8 @@ deploy:
   provider: pypi
   distributions: sdist bdist_wheel
   skip_existing: true
+  on:
+    branch: master
   user: "dvatterott"
   password:
     secure: Edplgu14/v2GjWCGe0NC+Jnn5oqtdbN/ydJNXVjWbqc9Qs10Y3QEF54QsKn4DFdYQD6sikX90k2mhXeH5wbRPN2RiuH2LWHvl5Dwr7wTZEIgnmoVReNo7+KsAc7l3Wqt75TJ7UH9MuBHBqkBkGEowC/Mj0rrgLgJMUIHlp329qD2UkglGRG8VkwdNDCBboSRE5fF3V2MfhtHlv1AR63gyjO7t1bchM/Rxkol0hGXFeEs1wUNC76oYpYAtearN56lGVVX44xMI04icPpwymuIpxv6Ej31hhaw9b/97IqM1p7j3R0uG4kepOYDBZ7SAMm79prmWyUdCbFR3e9QQNlgBGfD5TrZwxEA371C9sG3P2joIAgl4QVT99dU+nfSOnflSnz8KRI13bZsmdhELwX25gc4n3Y4mx+Fp8ZdCXUYiZCm+eKP3oYgPgHXnu1Tg3FG1aNRKFWIyvBQBTRIVohNjrRH+fUDozO63bwO3yfbYwN1MRLbjc+Pq7xudmoqo+J03ljMjglJLL1dihAtFOwWLIf8MyEqz9mQ0kgWZM8e1yZpB5zbE75q8MM/DTN49RokjEMvtCArCpa9hLJ4eIu4vtsjYEyex/Ypg2lmfDoFX9yY1/9GZBWo2oxpuKd6iH5h6REQ95EhVuIl5BlrXMHwS2ytKFduIlblCN7yXYKe8q8=

README.rst

@@ -9,6 +9,10 @@ predeval
 .. image:: https://img.shields.io/travis/dvatterott/predeval.svg
         :target: https://travis-ci.org/dvatterott/predeval
 
+.. |appveyor| image:: https://ci.appveyor.com/api/projects/status/github/dvatterott/predeval?branch=master&svg=true
+    :alt: AppVeyor Build Status
+    :target: https://ci.appveyor.com/project/dvatterott/predeval
+
 .. image:: https://readthedocs.org/projects/predeval/badge/?version=latest
         :target: https://predeval.readthedocs.io/en/latest/?badge=latest
         :alt: Documentation Status
@@ -20,10 +24,9 @@ predeval
      :target: https://pyup.io/repos/github/dvatterott/predeval/
      :alt: Updates
 
-.. image:: https://pyup.io/repos/github/dvatterott/predeval/python-3-shield.svg
-    :target: https://pyup.io/repos/github/dvatterott/predeval/
-    :alt: Python 3
-
+.. |supported-versions| image:: https://img.shields.io/pypi/pyversions/predeval.svg
+   :alt: Supported versions
+   :target: https://pypi.python.org/pypi/predeval
 
 
 This software is built to identify changes in a model output before evaluation data becomes available.

appveyor.yml

@@ -0,0 +1,36 @@
+branches:
+  only:
+    - master
+version: '{branch}-{build}'
+build: off
+cache:
+  - '%LOCALAPPDATA%\pip\Cache'
+environment:
+  matrix:
+    - TOXENV: 'py27'
+    - TOXENV: 'py34'
+    - TOXENV: 'py35'
+    - TOXENV: 'py36'
+
+init:
+  - ps: echo $env:TOXENV
+  - ps: ls C:\Python*
+install:
+  # install pypy using choco (redirect to a file and write to console in case
+  # choco install returns non-zero, because choco install python.pypy is too
+  # noisy)
+  - choco install python.pypy > pypy-inst.log 2>&1 || (type pypy-inst.log & exit /b 1)
+  - set PATH=C:\tools\pypy\pypy;%PATH% # so tox can find pypy
+  - echo PyPy installed
+  - pypy --version
+
+  - C:\Python35\python -m pip install tox
+
+test_script:
+  - C:\Python35\python -m tox
+
+on_failure:
+  - ps: dir "env:"
+  - ps: get-content .tox\*\log\*
+artifacts:
+- path: dist\*

docs/index.rst

@@ -1,7 +1,9 @@
 Welcome to predeval's documentation!
 ======================================
 
-This software is built to identify unexpected changes in a model output before evaluation data becomes available. For example, if you create a churn model, you will have to wait X number of weeks before learning whether users churned (and can evaluate your churn model predictions). This software will not guarantee that your model is accurate, but it will alert you if your model's outputs (i.e., predictions) are dramatically different from what they have been in the past.
+This software is built to identify unexpected changes in a model output before evaluation data becomes available.
+
+For example, if you create a churn model, you will have to wait X number of weeks before learning whether users churned (and can evaluate your churn model predictions). This software will not guarantee that your model is accurate, but it will alert you if your model's outputs (i.e., predictions) are dramatically different from what they have been in the past.
 
 .. toctree::
    :maxdepth: 2

predeval/__init__.py

@@ -4,7 +4,7 @@
 
 __author__ = """Dan Vatterott"""
 __email__ = 'dvatterott@gmail.com'
-__version__ = '0.0.3'
+__version__ = '0.0.4'
 
 from .continuous import ContinuousEvaluator
 from .categorical import CategoricalEvaluator

predeval/categorical.py

@@ -85,8 +85,8 @@ def __init__(
 
         # ---- Fill in Assertion Parameters ---- #
         self._assertion_params_ = {
-            'chi2_test': None,
             'cat_exists': None,
+            'chi2_test': None,
         }
 
         assert isinstance(kwargs.get('chi2_stat', 2),
@@ -95,12 +95,12 @@ def __init__(
 
         # ---- create list of assertions to test ---- #
         self._possible_assertions_ = {
-            'chi2_test': (self.update_chi2_test, self.check_chi2),
             'exist': (self.update_exist, self.check_exist),
+            'chi2_test': (self.update_chi2_test, self.check_chi2),
         }
 
         # ---- create list of assertions to test ---- #
-        assertions = ['chi2_test', 'exist'] if assertions is None else assertions
+        assertions = ['exist', 'chi2_test'] if assertions is None else assertions
         self._assertions_ = self._check_assertion_types(assertions)
 
         # ---- populate assertion tests with reference data ---- #

predeval/continuous.py

@@ -69,9 +69,9 @@ def __init__(
             'ks_test': None
         }
 
-        assert isinstance(kwargs.get('ks_stat', 0.2),
+        assert isinstance(kwargs.get('ks_stat', 0.5),
                           Real), 'expected number, input ks_test_stat is not a number'
-        self._assertion_params_['ks_stat'] = kwargs.get('ks_stat', 0.2)
+        self._assertion_params_['ks_stat'] = kwargs.get('ks_stat', 0.5)
 
         # ---- create list of assertions to test ---- #
         self._possible_assertions_ = {

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.3
+current_version = 0.0.4
 commit = True
 tag = True
 

setup.py

@@ -48,6 +48,6 @@
     test_suite='tests',
     tests_require=TEST_REQUIREMENTS,
     url='https://github.com/dvatterott/predeval',
-    version='0.0.3',
+    version='0.0.4',
     zip_safe=False,
 )

tests/test_predeval.py

@@ -41,16 +41,16 @@ def test_std_param(self):
 
     def test_kstest_param(self):
         """Assert that correctly setting ks_stat."""
-        assert self.con_eval.assertion_params['ks_stat'] == 0.2
+        assert self.con_eval.assertion_params['ks_stat'] == 0.5
 
     def test_kstest(self, capsys):
         """Assert that check_ks correct."""
         self.con_eval.check_ks(np.array([x for x in range(31)]))
         captured = capsys.readouterr()
         assert captured.out == "Passed ks check; test statistic=0.0000, p=1.0000\n"
-        self.con_eval.check_ks(np.array([x for x in range(51)]))
+        self.con_eval.check_ks(np.array([x for x in range(20, 51)]))
         captured = capsys.readouterr()
-        assert captured.out == "Failed ks check; test statistic=0.3922, p=0.0036\n"
+        assert captured.out == "Failed ks check; test statistic=0.6452, p=0.0000\n"
 
     def test_checkmin(self, capsys):
         """Assert that check_min correct."""
@@ -90,10 +90,10 @@ def test_checkstd(self, capsys):
 
     def test_update_ks_test(self, capsys):
         """Assert that correctly updating ks_test."""
-        self.con_eval.update_ks_test(np.array([x for x in range(51)]))
+        self.con_eval.update_ks_test(np.array([x for x in range(20, 51)]))
         self.con_eval.check_ks(np.array([x for x in range(31)]))
         captured = capsys.readouterr()
-        assert captured.out == "Failed ks check; test statistic=0.3922, p=0.0036\n"
+        assert captured.out == "Failed ks check; test statistic=0.6452, p=0.0000\n"
 
     def test_update_min(self):
         """Assert that correctly updating min."""
@@ -117,6 +117,7 @@ def test_update_std(self):
 
     def test_check_data(self, capsys):
         """Assert that correctly using check_data."""
+        self.con_eval.update_ks_test(np.array([x for x in range(51)]))
         self.con_eval.check_data(np.array([x for x in range(51)]))
         captured = capsys.readouterr()
         expect_out = ("Passed min check; min observed=0.0000\n"
@@ -223,8 +224,8 @@ def test_check_data(self, capsys):
         self.con_eval = CategoricalEvaluator(new_out)
         self.con_eval.check_data(new_out)
         captured = capsys.readouterr()
-        expect_out = ("Passed chi2 check; test statistic=0.0000, p=1.0000\n"
-                      "Passed exist check; observed=[0 1 2] (Expected [0, 1, 2])\n")
+        expect_out = ("Passed exist check; observed=[0 1 2] (Expected [0, 1, 2])\n"
+                      "Passed chi2 check; test statistic=0.0000, p=1.0000\n")
         assert captured.out == expect_out
 
 
@@ -241,8 +242,8 @@ def test_eval_tests(self, capsys):
         output = self.con_eval.check_data(new_out)
         evaluate_tests(output)
         captured = capsys.readouterr()
-        expect_out = ("Passed chi2 test.\n"
-                      "Passed exist test.\n")
+        expect_out = ("Passed exist test.\n"
+                      "Passed chi2 test.\n")
         assert captured.out == expect_out
 
     def test_eval_tests_fail(self, capsys):
@@ -253,8 +254,8 @@ def test_eval_tests_fail(self, capsys):
         output = self.con_eval.check_data(new_out)
         evaluate_tests(output)
         captured = capsys.readouterr()
-        expect_out = ("Failed chi2 test.\n"
-                      "Passed exist test.\n")
+        expect_out = ("Passed exist test.\n"
+                      "Failed chi2 test.\n")
         assert captured.out == expect_out
 
     def test_eval_tests_noprint(self, capsys):

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27, py34, py35, py36, flake8
+envlist = py27, py34, py35, py36, flake8, pypy, pypy3
 
 [travis]
 python =