Merge 6d6d33f into ae697f0

.travis.yml

@@ -20,8 +20,10 @@ before_install:
   - conda update -q conda
   - pip install coveralls
 install:
-  - conda install -q python=$TRAVIS_PYTHON_VERSION --file requirements.txt
-  - travis_retry python setup.py dev
+  - conda install -q python=$TRAVIS_PYTHON_VERSION
+  - pip install -r requirements.txt -r requirements-dev.txt
+  - pip install pandas==0.23.4  # for rpy2 compatibility
+  - pip install -e .[testing]
 script:
   - travis_wait py.test --runslow
 deploy:

examples/iris/config.py

@@ -35,6 +35,8 @@
     'model': {
         '__factory__': 'sklearn.linear_model.LogisticRegression',
         'C': 0.3,
+        'solver': 'lbfgs',
+        'multi_class': 'auto',
     },
 
     'grid_search': {

palladium/persistence.py

@@ -135,7 +135,7 @@ def open(self, path, mode='r'):
             else:
                 reader = codecs.getreader(res.encoding or 'utf-8')
                 return reader(res.raw)
-        elif mode[0] == 'w':
+        elif mode == 'wb':
             return self._write(path, mode=mode)
         raise NotImplementedError("filemode: %s" % (mode,))
 
@@ -252,8 +252,9 @@ def _read_md(self):
     def _update_md(self, data):
         data2 = self._read_md()
         data2.update(data)
-        with self.io.open(self._md_filename, 'w') as f:
-            json.dump(data2, f, indent=4)
+        with self.io.open(self._md_filename, 'wb') as f:
+            bytes = json.dumps(data2, indent=4).encode('utf-8')
+            f.write(bytes)
 
     def upgrade(self, from_version=None, to_version=__version__):
         if from_version is None:

palladium/tests/test_fit.py

@@ -438,6 +438,7 @@ def estimator(self):
             LogisticRegression(solver='liblinear'),
             param_grid={'C': [0.001, 0.01]},
             cv=3,
+            iid=False,
             )
 
     @pytest.mark.parametrize('backend', ['threading', 'sequential'])

palladium/tests/test_persistence.py

@@ -214,20 +214,21 @@ def test_list_properties_with_metadata(self, File):
 
     def test_update_md(self, File):
         with patch('palladium.persistence.File._read_md') as read_md,\
-            patch('palladium.persistence.json.dump') as dump,\
             patch('builtins.open') as open:
             read_md.return_value = {
                 'hello': 'world',
                 'models': [1],
                 'properties': {},
                 }
             File('model-{version}')._update_md_orig({'models': [2]})
-            open.assert_called_with('model-metadata.json', 'w')
-            dump.assert_called_with(
-                {'hello': 'world', 'models': [2], 'properties': {}},
-                open.return_value.__enter__.return_value,
-                indent=4,
-                )
+            open.assert_called_with('model-metadata.json', 'wb')
+            fh = open.return_value.__enter__.return_value
+            json_written = json.loads(fh.write.call_args[0][0].decode('utf-8'))
+            assert json_written == {
+                'hello': 'world',
+                'models': [2],
+                'properties': {},
+                }
 
     def test_read_md(self, File):
         with patch('builtins.open') as open,\
@@ -657,7 +658,7 @@ def handle_put_md(request, context):
         assert put_md.called
 
         assert pickle.loads(gzip.decompress(put_model_body)) == model
-        assert len(json.loads(put_md_body)['models']) == 1
+        assert len(json.loads(put_md_body.decode('utf-8'))['models']) == 1
         self.assert_auth_headers(mocked_requests)
 
     def test_download(self, mocked_requests, persister):
@@ -722,7 +723,7 @@ def handle_put_md(request, context):
         persister.delete(1)
         assert put_md.called
         assert delete_model.called
-        assert len(json.loads(put_md_body)['models']) == 0
+        assert len(json.loads(put_md_body.decode('utf-8'))['models']) == 0
         self.assert_auth_headers(mocked_requests)
 
 

palladium/tests/test_server.py

@@ -665,7 +665,7 @@ def test_it(self, fit, config, jobs, flask_app):
         config['model_persister'] = model_persister
         with flask_app.test_request_context(method='POST'):
             resp = fit()
-        sleep(0.005)
+        sleep(0.02)
         resp_json = json.loads(resp.get_data(as_text=True))
         job = jobs[resp_json['job_id']]
         assert job['status'] == 'finished'
@@ -686,7 +686,7 @@ def test_pass_args(self, fit, flask_app, args, args_expected):
             fit_base.__name__ = 'mock'
             with flask_app.test_request_context(method='POST', data=args):
                 fit()
-            sleep(0.005)
+            sleep(0.02)
         assert fit_base.call_args == call(**args_expected)
 
 
@@ -707,7 +707,7 @@ def test_success(self, update_model_cache, config, jobs, flask_app):
         config['model_persister'] = model_persister
         with flask_app.test_request_context(method='POST'):
             resp = update_model_cache()
-        sleep(0.005)
+        sleep(0.02)
         resp_json = json.loads(resp.get_data(as_text=True))
         job = jobs[resp_json['job_id']]
         assert job['status'] == 'finished'

palladium/tests/test_util.py

@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from datetime import datetime
 import threading
 from time import sleep
@@ -150,7 +151,7 @@ def test_last_execution(self, RruleThread):
             dtstart=datetime(2014, 10, 30, 13, 21, 18)))
         thread.last_execution = datetime(2014, 10, 30, 13, 21, 18)
         thread.start()
-        sleep(0.005)
+        sleep(0.02)
         assert func.call_count == 1
 
     def test_func_raises(self, RruleThread):
@@ -164,7 +165,7 @@ def test_func_raises(self, RruleThread):
 
         with patch('palladium.util.logger') as logger:
             thread.start()
-            sleep(0.005)
+            sleep(0.02)
             assert func.call_count == 1
             assert logger.exception.call_count == 1
 
@@ -174,7 +175,7 @@ def test_sleep_between_checks(self, RruleThread):
         rr.between.return_value = False
         thread = RruleThread(func, rr, sleep_between_checks=0.0010)
         thread.start()
-        sleep(0.005)
+        sleep(0.02)
         assert func.call_count == 0
         assert rr.between.call_count > 1
 
@@ -390,7 +391,7 @@ def myfunc(add):
         results = []
         for i in range(3):
             results.append(run_job(myfunc, add=i))
-        sleep(0.005)
+        sleep(0.02)
         assert result == 3
         assert len(jobs) == len(results) == 3
         assert set(jobs.keys()) == set(r[1] for r in results)
@@ -406,7 +407,7 @@ def myfunc(divisor):
         num_threads_before = len(threading.enumerate())
         for i in range(3):
             run_job(myfunc, divisor=i)
-        sleep(0.005)
+        sleep(0.02)
         num_threads_after = len(threading.enumerate())
 
         assert num_threads_before == num_threads_after
@@ -426,15 +427,21 @@ def myfunc(tts):
             run_job(myfunc, tts=i/100)
 
         job1, job2, job3 = sorted(jobs.values(), key=lambda x: x['started'])
-        assert job1['status'] == 'finished'
-        assert job2['status'] == job3['status'] == 'running'
-        assert len(threading.enumerate()) - num_threads_before == 2
 
-        sleep(0.015)
-        assert job2['status'] == 'finished'
-        assert job3['status'] == 'running'
-        assert len(threading.enumerate()) - num_threads_before == 1
-
-        sleep(0.015)
-        assert job3['status'] == 'finished'
-        assert len(threading.enumerate()) - num_threads_before == 0
+        samples = []
+        for i in range(10):
+            samples.append((
+                job1['status'],
+                job2['status'],
+                job3['status'],
+                len(threading.enumerate()),
+                ))
+            sleep(1/100)
+
+        got = list(OrderedDict.fromkeys(samples))
+        expected = [
+            ('finished', 'running', 'running', num_threads_before+2),
+            ('finished', 'finished', 'running', num_threads_before+1),
+            ('finished', 'finished', 'finished', num_threads_before+0),
+        ]
+        assert got == expected

requirements-dev.txt

@@ -1,6 +1,6 @@
 cov-core==1.15.0
-coverage==4.5.2
-pluggy==0.8.0
-pytest==4.0.1
-pytest-cov==2.6.0
-requests-mock==1.5.2
+coverage==4.5.4
+pluggy==0.12.0
+pytest==5.0.1
+pytest-cov==2.7.1
+requests-mock==1.6.0

requirements.txt

@@ -1,26 +1,24 @@
-attrs==18.2.0
-certifi==2018.11.29
+certifi==2019.6.16
 chardet==3.0.4
-click==7.0
+Click==7.0
 docopt==0.6.2
-flask==1.0.2
-idna==2.7
+Flask==1.1.1
+idna==2.8
 itsdangerous==1.1.0
-Jinja2==2.10
-joblib==0.13.0
-MarkupSafe==1.1.0
-more-itertools==4.3.0
-numpy==1.15.4
-pandas==0.23.4
-psutil==5.4.8
-python-dateutil==2.7.5
-pytz==2018.7
-requests==2.20.1
-scikit-learn==0.20.1
-scipy==1.1.0
+Jinja2==2.10.1
+joblib==0.13.2
+MarkupSafe==1.1.1
+numpy==1.17.0
+pandas==0.24.2
+psutil==5.6.3
+python-dateutil==2.8.0
+pytz==2019.2
+requests==2.22.0
+scikit-learn==0.21.3
+scipy==1.3.0
 six==1.12.0
-SQLAlchemy==1.2.14
+SQLAlchemy==1.3.6
 ujson==1.35
-urllib3==1.23
-Werkzeug==0.14.1
+urllib3==1.25.3
+Werkzeug==0.15.5
 # julia==0.4.5

setup.cfg

@@ -8,3 +8,5 @@ addopts =
   --cov=palladium --cov-report=term-missing --cov-config .coveragerc
   palladium/ examples/
 python_files = test*py
+markers =
+    slow: run slow tests