From 689c00379c6906f83cb3e867cba02e6913a799d2 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Fri, 1 Dec 2017 07:42:18 -0600 Subject: [PATCH 01/33] Remove errant docstring --- pyramid/arima/arima.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyramid/arima/arima.py b/pyramid/arima/arima.py index 6cbfda026..b12afabec 100644 --- a/pyramid/arima/arima.py +++ b/pyramid/arima/arima.py @@ -360,9 +360,6 @@ def predict_in_sample(self, exogenous=None, start=None, def predict(self, n_periods=10, exogenous=None): """Generate predictions (forecasts) ``n_periods`` in the future. - Note that unless ``include_std_err`` or ``include_conf_int`` are True, - only the forecast array will be returned (otherwise, a tuple with the - corresponding elements will be returned). Parameters ---------- From 2dcb63c6b9322440bd2b3342b391c482c5b1c523 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Fri, 1 Dec 2017 07:51:42 -0600 Subject: [PATCH 02/33] [DOC] :books: Update img.avatar in CSS to force contributor avatars to 60x60 --- doc/_static/css/gitcontrib.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/_static/css/gitcontrib.css b/doc/_static/css/gitcontrib.css index 7f6ad229d..dff0a6623 100644 --- a/doc/_static/css/gitcontrib.css +++ b/doc/_static/css/gitcontrib.css @@ -32,6 +32,11 @@ margin-top: 10px; } +img.avatar { + width: 60px; + height: 60px; +} + .contrib-author-wrapper { position: absolute; display: block; From 488e357748bee1fbd6608c02ef8e3f85cfb7369c Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Thu, 4 Jan 2018 11:13:32 -0600 Subject: [PATCH 03/33] Cut out SVD computation of U, Vt and watch how much faster this badboy moves! Also amended fetchContributor function in doc --- .gitignore | 3 +++ ISSUE_TEMPLATE.md | 2 +- doc/_static/js/contrib.js | 9 ++++++--- doc/contributors.rst | 4 ++++ pyramid/arima/arima.py | 7 ++++++- pyramid/arima/seasonality.py | 8 ++++++-- pyramid/arima/stationarity.py | 8 ++++---- 7 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 07deaa1be..8deacf27e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ # Pypi pws .pypipws +# Coverage +coverage + # OAuth access for automating releases ACCESS_TOKEN diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index f006d3b67..4c76e454c 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -1,7 +1,7 @@ #### Description - + #### Steps/Code to Reproduce +
    \ No newline at end of file diff --git a/pyramid/arima/arima.py b/pyramid/arima/arima.py index b12afabec..4c2f13dd2 100644 --- a/pyramid/arima/arima.py +++ b/pyramid/arima/arima.py @@ -215,6 +215,9 @@ def fit(self, y, exogenous=None, **fit_args): operation. This should not include a constant or trend. Note that if an ``ARIMA`` is fit on exogenous features, it must be provided exogenous features for making predictions. + + **fit_args : dict or kwargs + Any keyword arguments to pass to the statsmodels ARIMA fit. """ y = c1d(check_array(y, ensure_2d=False, force_all_finite=False, copy=True, dtype=DTYPE)) # type: np.ndarray @@ -360,6 +363,8 @@ def predict_in_sample(self, exogenous=None, start=None, def predict(self, n_periods=10, exogenous=None): """Generate predictions (forecasts) ``n_periods`` in the future. + Note that if ``exogenous`` variables were used in the model fit, they + will be expected for the predict procedure and will fail otherwise. Parameters ---------- @@ -420,7 +425,7 @@ def fit_predict(self, y, exogenous=None, n_periods=10, **fit_args): n_periods : int, optional (default=10) The number of periods in the future to forecast. - fit_args : dict, optional (default=None) + fit_args : dict or kwargs, optional (default=None) Any keyword args to pass to the fit method. """ self.fit(y, exogenous, **fit_args) diff --git a/pyramid/arima/seasonality.py b/pyramid/arima/seasonality.py index 57c73837f..ee5c74aa6 100644 --- a/pyramid/arima/seasonality.py +++ b/pyramid/arima/seasonality.py @@ -130,7 +130,8 @@ def _sd_test(wts, s): frecob = np.zeros(s - 1).astype('int64') # I hate looping like this, but it seems like overkill to - # write a C function for something that's otherwise so trivial... + # write a C function for something that's otherwise so trivial + # (especially due to the overhead of calling a C func from python) for i, v in enumerate(frec): if v == 1 and i == int(s / 2) - 1: frecob[sq[i]] = 1 @@ -146,7 +147,10 @@ def _sd_test(wts, s): C_pop_A(A, frecob) tmp = A.T.dot(Omfhat).dot(A) - _, sv, _ = svd(tmp) + + # UPDATE 01/04/2018 - we can get away without computing u, v + # (this is also MUCH MUCH faster!!!) + sv = svd(tmp, compute_uv=False) if sv.min() < 2.220446e-16: # machine min eps return 0 diff --git a/pyramid/arima/stationarity.py b/pyramid/arima/stationarity.py index 0a619d68b..22a7fc7f3 100644 --- a/pyramid/arima/stationarity.py +++ b/pyramid/arima/stationarity.py @@ -147,10 +147,10 @@ def is_stationary(self, x): scalar, denom = 10, 14 if self.lshort: scalar, denom = 3, 13 - l = int(np.trunc(scalar * np.sqrt(n) / denom)) + l_ = int(np.trunc(scalar * np.sqrt(n) / denom)) # compute the C subroutine - s2 = C_tseries_pp_sum(e, n, l, s2) + s2 = C_tseries_pp_sum(e, n, l_, s2) stat = eta / s2 # do approximation @@ -345,8 +345,8 @@ def is_stationary(self, x): ssqru = (u * u).sum() / float(n) scalar = 12 if not self.lshort else 4 - l = int(np.trunc(scalar * np.power(n / 100.0, 0.25))) - ssqrtl = C_tseries_pp_sum(u, n, l, ssqru) + l_ = int(np.trunc(scalar * np.power(n / 100.0, 0.25))) + ssqrtl = C_tseries_pp_sum(u, n, l_, ssqru) # define trm vals n2 = n * n From 0e12c3b4c4b128f2e8c256e69a41556fd8244f23 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Thu, 4 Jan 2018 11:20:05 -0600 Subject: [PATCH 04/33] [ci skip][DOC] :books: update documentation and quickstart --- doc/quickstart.rst | 36 ++++++++++++++++++------------------ doc/setup.rst | 3 +++ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/doc/quickstart.rst b/doc/quickstart.rst index 03740bd45..e4890a76b 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -20,26 +20,26 @@ Here is a simple example of Pyramid use: ... error_action='ignore', # don't want to know if an order does not work ... suppress_warnings=True, # don't want convergence warnings ... stepwise=True) # set to stepwise - Fit ARIMA: order=(1, 1, 1) seasonal_order=(0, 1, 1, 12); AIC=3066.811, BIC=3082.663, Fit time=1.731 seconds + Fit ARIMA: order=(1, 1, 1) seasonal_order=(0, 1, 1, 12); AIC=3066.811, BIC=3082.663, Fit time=0.517 seconds Fit ARIMA: order=(0, 1, 0) seasonal_order=(0, 1, 0, 12); AIC=nan, BIC=nan, Fit time=nan seconds - Fit ARIMA: order=(1, 1, 0) seasonal_order=(1, 1, 0, 12); AIC=3099.735, BIC=3112.417, Fit time=0.434 seconds - Fit ARIMA: order=(0, 1, 1) seasonal_order=(0, 1, 1, 12); AIC=3066.983, BIC=3079.665, Fit time=0.473 seconds - Fit ARIMA: order=(1, 1, 1) seasonal_order=(1, 1, 1, 12); AIC=3067.666, BIC=3086.688, Fit time=2.184 seconds - Fit ARIMA: order=(1, 1, 1) seasonal_order=(0, 1, 0, 12); AIC=3088.109, BIC=3100.791, Fit time=0.344 seconds - Fit ARIMA: order=(1, 1, 1) seasonal_order=(0, 1, 2, 12); AIC=3067.669, BIC=3086.692, Fit time=4.371 seconds - Fit ARIMA: order=(1, 1, 1) seasonal_order=(1, 1, 2, 12); AIC=3068.757, BIC=3090.951, Fit time=4.160 seconds - Fit ARIMA: order=(2, 1, 1) seasonal_order=(0, 1, 1, 12); AIC=3067.485, BIC=3086.508, Fit time=1.204 seconds - Fit ARIMA: order=(1, 1, 0) seasonal_order=(0, 1, 1, 12); AIC=3094.578, BIC=3107.260, Fit time=0.468 seconds - Fit ARIMA: order=(1, 1, 2) seasonal_order=(0, 1, 1, 12); AIC=3066.771, BIC=3085.794, Fit time=1.222 seconds - Fit ARIMA: order=(2, 1, 3) seasonal_order=(0, 1, 1, 12); AIC=3070.642, BIC=3096.006, Fit time=3.495 seconds - Fit ARIMA: order=(1, 1, 2) seasonal_order=(1, 1, 1, 12); AIC=3068.086, BIC=3090.280, Fit time=1.180 seconds - Fit ARIMA: order=(1, 1, 2) seasonal_order=(0, 1, 0, 12); AIC=3090.977, BIC=3106.830, Fit time=0.544 seconds - Fit ARIMA: order=(1, 1, 2) seasonal_order=(0, 1, 2, 12); AIC=3067.766, BIC=3089.959, Fit time=3.303 seconds - Fit ARIMA: order=(1, 1, 2) seasonal_order=(1, 1, 2, 12); AIC=3069.717, BIC=3095.081, Fit time=5.675 seconds + Fit ARIMA: order=(1, 1, 0) seasonal_order=(1, 1, 0, 12); AIC=3099.735, BIC=3112.417, Fit time=0.162 seconds + Fit ARIMA: order=(0, 1, 1) seasonal_order=(0, 1, 1, 12); AIC=3066.983, BIC=3079.665, Fit time=0.164 seconds + Fit ARIMA: order=(1, 1, 1) seasonal_order=(1, 1, 1, 12); AIC=3067.666, BIC=3086.688, Fit time=0.645 seconds + Fit ARIMA: order=(1, 1, 1) seasonal_order=(0, 1, 0, 12); AIC=3088.109, BIC=3100.791, Fit time=0.136 seconds + Fit ARIMA: order=(1, 1, 1) seasonal_order=(0, 1, 2, 12); AIC=3067.669, BIC=3086.692, Fit time=1.512 seconds + Fit ARIMA: order=(1, 1, 1) seasonal_order=(1, 1, 2, 12); AIC=3068.757, BIC=3090.951, Fit time=1.651 seconds + Fit ARIMA: order=(2, 1, 1) seasonal_order=(0, 1, 1, 12); AIC=3067.485, BIC=3086.508, Fit time=0.445 seconds + Fit ARIMA: order=(1, 1, 0) seasonal_order=(0, 1, 1, 12); AIC=3094.578, BIC=3107.260, Fit time=0.174 seconds + Fit ARIMA: order=(1, 1, 2) seasonal_order=(0, 1, 1, 12); AIC=3066.771, BIC=3085.794, Fit time=0.425 seconds + Fit ARIMA: order=(2, 1, 3) seasonal_order=(0, 1, 1, 12); AIC=3070.642, BIC=3096.006, Fit time=0.966 seconds + Fit ARIMA: order=(1, 1, 2) seasonal_order=(1, 1, 1, 12); AIC=3068.086, BIC=3090.280, Fit time=0.411 seconds + Fit ARIMA: order=(1, 1, 2) seasonal_order=(0, 1, 0, 12); AIC=3090.977, BIC=3106.830, Fit time=0.249 seconds + Fit ARIMA: order=(1, 1, 2) seasonal_order=(0, 1, 2, 12); AIC=3067.766, BIC=3089.959, Fit time=1.170 seconds + Fit ARIMA: order=(1, 1, 2) seasonal_order=(1, 1, 2, 12); AIC=3069.717, BIC=3095.081, Fit time=2.000 seconds Fit ARIMA: order=(0, 1, 2) seasonal_order=(0, 1, 1, 12); AIC=nan, BIC=nan, Fit time=nan seconds - Fit ARIMA: order=(2, 1, 2) seasonal_order=(0, 1, 1, 12); AIC=3068.701, BIC=3090.895, Fit time=1.509 seconds - Fit ARIMA: order=(1, 1, 3) seasonal_order=(0, 1, 1, 12); AIC=3068.842, BIC=3091.036, Fit time=2.329 seconds - Total fit time: 34.636 seconds + Fit ARIMA: order=(2, 1, 2) seasonal_order=(0, 1, 1, 12); AIC=3068.701, BIC=3090.895, Fit time=0.523 seconds + Fit ARIMA: order=(1, 1, 3) seasonal_order=(0, 1, 1, 12); AIC=3068.842, BIC=3091.036, Fit time=0.590 seconds + Total fit time: 11.745 seconds It's easy to examine your model fit results. Simply use the ``summary`` method: diff --git a/doc/setup.rst b/doc/setup.rst index 6d90d4e24..e27564d49 100644 --- a/doc/setup.rst +++ b/doc/setup.rst @@ -17,3 +17,6 @@ python: .. code-block:: python from pyramid.arima import auto_arima + +If you encounter an ``ImportError``, try updating numpy and re-installing. Outdated +numpy versions have been observed to break the Pyramid build. From 258a28559d07aad945ac2d93f3ca2bfd6deb1125 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Thu, 4 Jan 2018 11:21:12 -0600 Subject: [PATCH 05/33] Bump version --- pyramid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyramid/__init__.py b/pyramid/__init__.py index 685558d9a..608565b38 100644 --- a/pyramid/__init__.py +++ b/pyramid/__init__.py @@ -4,7 +4,7 @@ # # The pyramid module -__version__ = "0.6.1-dev0" +__version__ = "0.6.5-dev" try: # this var is injected in the setup build to enable From 86e95575f21608afcfacc3086614bb5329c183c2 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Thu, 4 Jan 2018 14:51:53 -0600 Subject: [PATCH 06/33] Fix contributor JS code --- doc/_static/js/contrib.js | 4 +--- doc/contributors.rst | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/_static/js/contrib.js b/doc/_static/js/contrib.js index ab7cacec9..6eded1cc6 100644 --- a/doc/_static/js/contrib.js +++ b/doc/_static/js/contrib.js @@ -55,9 +55,7 @@ function fetchContributors() { '') // can only do this once the doc is ready - $(document).ready(function() { - $('#contrib').append(li); - }); + $('#contrib').append(li); }); }); } diff --git a/doc/contributors.rst b/doc/contributors.rst index d1884d4e9..1889a570a 100644 --- a/doc/contributors.rst +++ b/doc/contributors.rst @@ -13,7 +13,9 @@ Thanks to the following users for their contributions to Pyramid! From 7306d80777e47ef2b775e2f68848a0c4d59db4d8 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Thu, 4 Jan 2018 14:56:38 -0600 Subject: [PATCH 07/33] [ci skip] Add author --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index cfbed380e..e562276d2 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -5,6 +5,7 @@ The following people have been core contributors to `pyramid`'s development: * [Taylor Smith](https://github.com/tgsmith61591) * [Gary Foreman](https://github.com/garyForeman) * [Charles Drotar](https://github.com/charlesdrotar) + * [Steven Hoelscher](https://github.com/shoelsch) __Please do not email the authors directly with questions or issues.__ Rather, use the [issues](https://github.com/tgsmith61591/pyramid/issues) page. Furthermore, issues From f6f7e0b0e78a3c307115304a015ec548a3214d17 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Thu, 4 Jan 2018 15:03:55 -0600 Subject: [PATCH 08/33] Trying to fix breaking appveyor build --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index cc874eefa..a46517ff6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -30,14 +30,14 @@ environment: PYTHON_VERSION: "3.5" build_script: - - ps: Start-FileDownload "https://repo.continuum.io/miniconda/Miniconda$env:PY_MAJOR_VER-latest-Windows-$env:PYTHON_ARCH.exe" C:\Miniconda.exe; echo "Finished downloading miniconda" + - cmd: Start-FileDownload "https://repo.continuum.io/miniconda/Miniconda$env:PY_MAJOR_VER-latest-Windows-$env:PYTHON_ARCH.exe" C:\Miniconda.exe; echo "Finished downloading miniconda" - cmd: C:\Miniconda.exe /S /D=C:\Py - SET PATH=C:\Py;C:\Py\Scripts;C:\Py\Library\bin;%PATH% - conda config --set always_yes yes - conda update conda --quiet # Things we'll need for build/test - - ps: conda create -n testenv --yes python=$env:PYTHON_VERSION numpy scipy cython scikit-learn statsmodels pip nose patsy + - cmd: conda create -n testenv --yes python=$env:PYTHON_VERSION numpy scipy cython scikit-learn statsmodels pip nose patsy - activate testenv # Things we'll need for deployment From 660ff5ad7a4c96a65d50d23539832d38b3188f16 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Thu, 4 Jan 2018 15:05:43 -0600 Subject: [PATCH 09/33] More broken appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index a46517ff6..95c6873d1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -30,7 +30,7 @@ environment: PYTHON_VERSION: "3.5" build_script: - - cmd: Start-FileDownload "https://repo.continuum.io/miniconda/Miniconda$env:PY_MAJOR_VER-latest-Windows-$env:PYTHON_ARCH.exe" C:\Miniconda.exe; echo "Finished downloading miniconda" + - ps: Start-FileDownload "https://repo.continuum.io/miniconda/Miniconda$env:PY_MAJOR_VER-latest-Windows-$env:PYTHON_ARCH.exe" C:\Miniconda.exe; echo "Finished downloading miniconda" - cmd: C:\Miniconda.exe /S /D=C:\Py - SET PATH=C:\Py;C:\Py\Scripts;C:\Py\Library\bin;%PATH% - conda config --set always_yes yes From 4137ae6240d89a4dd65d43699f525cef7068fd6a Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Thu, 4 Jan 2018 15:08:27 -0600 Subject: [PATCH 10/33] a;sdofjkl --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 95c6873d1..cc874eefa 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -37,7 +37,7 @@ build_script: - conda update conda --quiet # Things we'll need for build/test - - cmd: conda create -n testenv --yes python=$env:PYTHON_VERSION numpy scipy cython scikit-learn statsmodels pip nose patsy + - ps: conda create -n testenv --yes python=$env:PYTHON_VERSION numpy scipy cython scikit-learn statsmodels pip nose patsy - activate testenv # Things we'll need for deployment From 0ca92af2a06104dc174a74f0f23b50471b22ba4d Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Thu, 4 Jan 2018 15:31:03 -0600 Subject: [PATCH 11/33] Travis is trippin' too?? --- build_tools/travis/before_install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build_tools/travis/before_install.sh b/build_tools/travis/before_install.sh index 61a8d13ef..0d2f23e4d 100755 --- a/build_tools/travis/before_install.sh +++ b/build_tools/travis/before_install.sh @@ -6,4 +6,7 @@ set -e if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then echo "Updating apt-get for Linux build" sudo apt-get -qq update +# Because screw you, Travis: +elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + rvm get stable fi From b68e15bb80c59294d5ecb3f677b1e4c232c9272e Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Thu, 4 Jan 2018 15:42:34 -0600 Subject: [PATCH 12/33] Powershell woes --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index cc874eefa..1752527f2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -37,7 +37,7 @@ build_script: - conda update conda --quiet # Things we'll need for build/test - - ps: conda create -n testenv --yes python=$env:PYTHON_VERSION numpy scipy cython scikit-learn statsmodels pip nose patsy + - conda create -n testenv --yes python=$env:PYTHON_VERSION numpy scipy cython scikit-learn statsmodels pip nose patsy - activate testenv # Things we'll need for deployment From 8ed21d4d2359a43a11cc82eab999fea65d56673c Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Fri, 5 Jan 2018 09:39:33 -0600 Subject: [PATCH 13/33] Define lambda functions outside loops to avoid overhead --- .travis.yml | 2 +- appveyor.yml | 2 +- build_tools/travis/before_install.sh | 4 ++-- pyramid/arima/approx.py | 12 +++++++++++- pyramid/arima/auto.py | 14 +++++++++----- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7192fb5c8..74e0137f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,7 +85,7 @@ install: source build_tools/travis/install.sh before_script: bash build_tools/travis/before_script.sh script: bash build_tools/travis/test_script.sh after_success: - - source build_tools/travis/after_success.sh + - source build_tools/travis/after_success.sh # || "shell_session_update failed" # Build the wheels every time so we can debug - bash build_tools/travis/build_wheels.sh - ls dist/ diff --git a/appveyor.yml b/appveyor.yml index 1752527f2..cc874eefa 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -37,7 +37,7 @@ build_script: - conda update conda --quiet # Things we'll need for build/test - - conda create -n testenv --yes python=$env:PYTHON_VERSION numpy scipy cython scikit-learn statsmodels pip nose patsy + - ps: conda create -n testenv --yes python=$env:PYTHON_VERSION numpy scipy cython scikit-learn statsmodels pip nose patsy - activate testenv # Things we'll need for deployment diff --git a/build_tools/travis/before_install.sh b/build_tools/travis/before_install.sh index 0d2f23e4d..e8cbf3a25 100755 --- a/build_tools/travis/before_install.sh +++ b/build_tools/travis/before_install.sh @@ -7,6 +7,6 @@ if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then echo "Updating apt-get for Linux build" sudo apt-get -qq update # Because screw you, Travis: -elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - rvm get stable +# elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then +# rvm get stable fi diff --git a/pyramid/arima/approx.py b/pyramid/arima/approx.py index 6ea312a27..0d4714421 100644 --- a/pyramid/arima/approx.py +++ b/pyramid/arima/approx.py @@ -33,6 +33,10 @@ 'mean': np.average } +# identity function defined once to avoid multiple lambda calls +# littered throughout +_identity = (lambda t: t) + def _regularize(x, y, ties): """Regularize the values, make them ordered and remove duplicates. @@ -82,8 +86,14 @@ def tie_apply(f, u_val): return f(vals) # replace the duplicates in the y array with the "tie" func - func = VALID_TIES.get(ties, lambda t: t) + func = VALID_TIES.get(ties, _identity) + + # maybe expensive to vectorize on the fly? Not sure; would need + # to do some benchmarking. However, we need to in order to keep y + # and x in scope... y = np.vectorize(tie_apply)(func, ux) + + # does ux need ordering? hmm.. x = ux return x, y diff --git a/pyramid/arima/auto.py b/pyramid/arima/auto.py index 74b66bf3a..bf39931a8 100644 --- a/pyramid/arima/auto.py +++ b/pyramid/arima/auto.py @@ -369,7 +369,7 @@ def auto_arima(y, exogenous=None, start_p=2, d=None, start_q=2, max_p=5, # copy array y = column_or_1d(check_array(y, ensure_2d=False, dtype=DTYPE, copy=True, - force_all_finite=True)) + force_all_finite=True)) # type: np.ndarray n_samples = y.shape[0] # check for constant data @@ -628,7 +628,8 @@ def auto_arima(y, exogenous=None, start_p=2, d=None, start_q=2, max_p=5, key=(lambda mod: getattr(mod, information_criterion)())) - # remove all the cached .pmdpkl files... + # remove all the cached .pmdpkl files... someday write this as an exit hook + # in case of a KeyboardInterrupt or anything for model in sorted_res: model._clear_cached_state() @@ -705,7 +706,11 @@ def __init__(self, y, xreg, start_params, trend, method, transparams, # results list to store intermittent hashes of orders to determine if # we've seen this order before... - self.results_dict = dict() + self.results_dict = dict() # type: dict[tuple, ARIMA] + + # define the info criterion getter ONCE to avoid multiple lambda + # creation calls + self.get_ic = (lambda mod: getattr(mod, self.information_criterion)()) def is_new_better(self, new_model): if self.bestfit is None: @@ -713,8 +718,7 @@ def is_new_better(self, new_model): elif new_model is None: return False - get_ic = (lambda m: getattr(m, self.information_criterion)()) - current_ic, new_ic = get_ic(self.bestfit), get_ic(new_model) + current_ic, new_ic = self.get_ic(self.bestfit), self.get_ic(new_model) return new_ic < current_ic # this function takes a boolean expression, fits & caches a model, From ee48614dc2685ebff481d00f5246f9a33bacb0e4 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Fri, 5 Jan 2018 09:58:47 -0600 Subject: [PATCH 14/33] Add conda debugging in appveyor yml --- appveyor.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index cc874eefa..e5af5f35e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -34,10 +34,12 @@ build_script: - cmd: C:\Miniconda.exe /S /D=C:\Py - SET PATH=C:\Py;C:\Py\Scripts;C:\Py\Library\bin;%PATH% - conda config --set always_yes yes - - conda update conda --quiet + - conda update conda + - ps: echo "Conda version:" + - conda --version # Things we'll need for build/test - - ps: conda create -n testenv --yes python=$env:PYTHON_VERSION numpy scipy cython scikit-learn statsmodels pip nose patsy + - conda create -n testenv --yes python=$env:PYTHON_VERSION numpy scipy cython scikit-learn statsmodels pip nose patsy - activate testenv # Things we'll need for deployment From 07f037b2eeb5f5d2862cbbbdb67db719a11ce0a9 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Fri, 5 Jan 2018 10:01:56 -0600 Subject: [PATCH 15/33] More appveyor work --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index e5af5f35e..efa15bf1a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -37,9 +37,10 @@ build_script: - conda update conda - ps: echo "Conda version:" - conda --version + - ps: echo "Python version=$env:PYTHON_VERSION" # Things we'll need for build/test - - conda create -n testenv --yes python=$env:PYTHON_VERSION numpy scipy cython scikit-learn statsmodels pip nose patsy + - ps: conda create -n testenv --yes python=$env:PYTHON_VERSION numpy scipy cython scikit-learn statsmodels pip nose patsy - activate testenv # Things we'll need for deployment From 4029c77c57a2454aef22bf3ca52800148ba7206f Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Fri, 5 Jan 2018 10:12:59 -0600 Subject: [PATCH 16/33] What if we don't update Conda? Might be a Miniconda problem. --- appveyor.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index efa15bf1a..746a96fac 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -34,7 +34,9 @@ build_script: - cmd: C:\Miniconda.exe /S /D=C:\Py - SET PATH=C:\Py;C:\Py\Scripts;C:\Py\Library\bin;%PATH% - conda config --set always_yes yes - - conda update conda + # - conda update conda + + # Debugging because Appveyor gives us no stack traces when this stuff breaks. - ps: echo "Conda version:" - conda --version - ps: echo "Python version=$env:PYTHON_VERSION" From 55fc876b9131bd190de2e10bdd29d57bfc028a1f Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Fri, 5 Jan 2018 19:29:48 -0600 Subject: [PATCH 17/33] Amend doc, add is_iterable func, compute machine eps dynamically --- benchmarks/benchmark_funcs.py | 57 +++++++++++++++++++++++++++++++ pyramid/arima/auto.py | 12 ++++--- pyramid/arima/seasonality.py | 16 +++++++-- pyramid/arima/utils.py | 1 + pyramid/utils/array.py | 51 +++++++++++++++++++++++---- pyramid/utils/metaestimators.py | 3 +- pyramid/utils/tests/test_array.py | 11 +++++- 7 files changed, 135 insertions(+), 16 deletions(-) create mode 100644 benchmarks/benchmark_funcs.py diff --git a/benchmarks/benchmark_funcs.py b/benchmarks/benchmark_funcs.py new file mode 100644 index 000000000..54e452fd6 --- /dev/null +++ b/benchmarks/benchmark_funcs.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# +# Benchmark various approaches to functions to speed things up. +# ... hopefully. + +from __future__ import absolute_import + +import numpy as np + +import time + + +def _do_time(func, n_iter=10, *args, **kwargs): + times = [] + for _ in range(n_iter): + start = time.time() + func(*args, **kwargs) + times.append(time.time() - start) + + times = np.asarray(times) + print("Completed %i iterations (avg=%.6f, min=%.6f, max=%.6f)" + % (n_iter, times.mean(), times.min(), times.max())) + + +def benchmark_is_constant(): + """This benchmarks the "is_constant" function from ``pyramid.arima.utils``. + This was added in 0.6.2. + """ + # WINNER! + def is_const1(x): + """This is the version in Pyramid 0.6.2. + + Parameters + ---------- + x : np.ndarray + This is the array. + """ + return (x == x[0]).all() + + def is_const2(x): + """This should ostensibly only take O(N) rather than O(2N) like + its predecessor. But we'll see... + + Parameters + ---------- + x : np.ndarray + This is the array. + """ + return np.unique(x).shape[0] == 1 + + x = np.random.choice(np.arange(10), 1000000, replace=True) + _do_time(is_const1, 25, x) + _do_time(is_const2, 25, x) + + +if __name__ == '__main__': + benchmark_is_constant() diff --git a/pyramid/arima/auto.py b/pyramid/arima/auto.py index bf39931a8..0e5d50cc3 100644 --- a/pyramid/arima/auto.py +++ b/pyramid/arima/auto.py @@ -5,17 +5,20 @@ # Automatically find optimal parameters for an ARIMA from __future__ import absolute_import + from sklearn.utils.validation import check_array, column_or_1d from sklearn.utils import check_random_state from sklearn.externals.joblib import Parallel, delayed from sklearn.linear_model import LinearRegression + from numpy.linalg import LinAlgError import numpy as np + import warnings import time from .utils import ndiffs, is_constant, nsdiffs -from ..utils import diff +from ..utils import diff, is_iterable from .arima import ARIMA # for python 3 compat @@ -874,8 +877,9 @@ def _post_ppc_arima(a): The list or ARIMAs, or an ARIMA """ # if it's a result of making it to the end, it will - # be a list of ARIMA models. - if hasattr(a, '__iter__'): + # be a list of ARIMA models. Filter out the Nones + # (the failed models)... + if is_iterable(a): a = [m for m in a if m is not None] # if the list is empty, or if it was an ARIMA and it's None @@ -906,7 +910,7 @@ def _return_wrapper(fits, return_all, start, trace): Whether to return all. """ # make sure it's an iterable - if not hasattr(fits, '__iter__'): + if not is_iterable(fits): fits = [fits] # whether to print the final runtime diff --git a/pyramid/arima/seasonality.py b/pyramid/arima/seasonality.py index ee5c74aa6..d530961cb 100644 --- a/pyramid/arima/seasonality.py +++ b/pyramid/arima/seasonality.py @@ -150,12 +150,22 @@ def _sd_test(wts, s): # UPDATE 01/04/2018 - we can get away without computing u, v # (this is also MUCH MUCH faster!!!) - sv = svd(tmp, compute_uv=False) - if sv.min() < 2.220446e-16: # machine min eps + sv = svd(tmp, compute_uv=False) # type: np.ndarray + + # From R: + # double.eps: the smallest positive floating-point number ‘x’ such that + # ‘1 + x != 1’. It equals ‘double.base ^ ulp.digits’ if either + # ‘double.base’ is 2 or ‘double.rounding’ is 0; otherwise, it + # is ‘(double.base ^ double.ulp.digits) / 2’. Normally + # ‘2.220446e-16’. + # Numpy's float64 has an eps of 2.2204460492503131e-16 + if sv.min() < np.finfo(sv.dtype).eps: # machine min eps return 0 # solve against the identity matrix, then produce - # a nasty mess of dot products... + # a nasty mess of dot products... this is the (horrendous) R code: + # (1/N^2) * sum(diag(solve(tmp) %*% t(A) %*% t(Fhat) %*% Fhat %*% A)) + # https://github.com/robjhyndman/forecast/blob/master/R/arima.R#L321 solved = solve(tmp, np.identity(tmp.shape[0])) return (1.0 / n ** 2) * solved.dot(A.T).dot( Fhat.T).dot(Fhat).dot(A).diagonal().sum() diff --git a/pyramid/arima/utils.py b/pyramid/arima/utils.py index 77f9ef416..76ca5aa6c 100644 --- a/pyramid/arima/utils.py +++ b/pyramid/arima/utils.py @@ -5,6 +5,7 @@ # Common ARIMA functions from __future__ import absolute_import + from sklearn.utils.validation import check_array, column_or_1d import numpy as np diff --git a/pyramid/utils/array.py b/pyramid/utils/array.py index 6b2c775f7..c58f26a87 100644 --- a/pyramid/utils/array.py +++ b/pyramid/utils/array.py @@ -5,12 +5,16 @@ # Array utilities from __future__ import absolute_import, division + from sklearn.utils.validation import check_array +from sklearn.externals import six + import numpy as np __all__ = [ 'c', - 'diff' + 'diff', + 'is_iterable' ] @@ -20,16 +24,27 @@ def c(*args): that wraps ``numpy.concatenate``? Similar to R, this works with scalars, iterables, and any mix therein. + Note that using the ``c`` function on multi-nested lists or iterables + will fail! + Examples -------- - >>> from pyramid.utils import c + Using ``c`` with *args will yield a single array: >>> c(1, 2, 3, 4) array([1, 2, 3, 4]) - >>> from pyramid.utils import c + Using ``c`` with nested lists and scalars will also yield a single array: >>> c([1, 2], 4, c(5, 4)) array([1, 2, 4, 5, 4]) + However, using ``c`` with multi-level lists will fail! + >>> c([1, 2, 3], [[1, 2]]) + Traceback (most recent call last): + File "", line 1, in + File "pyramid/utils/array.py", line 64, in c + return np.concatenate([a if is_iterable(a) else [a] for a in args]) + ValueError: all the input arrays must have same number of dimensions + References ---------- .. [1] https://stat.ethz.ch/R-manual/R-devel/library/base/html/c.html @@ -43,14 +58,14 @@ def c(*args): element = args[0] # if it's iterable, make it an array - if hasattr(element, '__iter__'): + if is_iterable(element): return np.asarray(element) # otherwise it's not iterable, put it in an array return np.asarray([element]) # concat all - return np.concatenate([a if hasattr(a, '__iter__') else [a] for a in args]) + return np.concatenate([a if is_iterable(a) else [a] for a in args]) def _diff_vector(x, lag): @@ -107,7 +122,7 @@ def diff(x, lag=1, differences=1): >>> diff(x, 6, 1) array([], dtype=float32) - >>> from pyramid.utils import c, diff + >>> from pyramid.utils import diff >>> import numpy as np >>> >>> x = np.arange(1, 10).reshape((3, 3)).T @@ -138,7 +153,7 @@ def diff(x, lag=1, differences=1): if any(v < 1 for v in (lag, differences)): raise ValueError('lag and differences must be positive (> 0) integers') - x = check_array(x, ensure_2d=False, dtype=np.float32) + x = check_array(x, ensure_2d=False, dtype=np.float32) # type: np.ndarray fun = _diff_vector if len(x.shape) == 1 else _diff_matrix res = x @@ -150,3 +165,25 @@ def diff(x, lag=1, differences=1): return res return res + + +def is_iterable(x): + """Determine whether an object ``x`` is iterable. In Python 2, this + was as simple as checking for the ``__iter__`` attribute. However, in + Python 3, strings became iterable. Therefore, this function checks for the + ``__iter__`` attribute, returning True if present (except for strings, + for which it will return False). + + Parameters + ---------- + x : str, iterable or object + The object in question. + + Returns + ------- + isiter : bool + True if iterable, else False. + """ + if isinstance(x, six.string_types): + return False + return hasattr(x, '__iter__') diff --git a/pyramid/utils/metaestimators.py b/pyramid/utils/metaestimators.py index 86b346230..b0304cb63 100644 --- a/pyramid/utils/metaestimators.py +++ b/pyramid/utils/metaestimators.py @@ -2,7 +2,8 @@ # # Author: Taylor Smith # -# Metaestimators for the ARIMA class +# Metaestimators for the ARIMA class. These classes are derived from the +# sklearn metaestimators, but adapted for more specific use with pyramid. from __future__ import absolute_import from operator import attrgetter diff --git a/pyramid/utils/tests/test_array.py b/pyramid/utils/tests/test_array.py index 88542fca2..b750c5374 100644 --- a/pyramid/utils/tests/test_array.py +++ b/pyramid/utils/tests/test_array.py @@ -1,7 +1,9 @@ from __future__ import absolute_import -from pyramid.utils.array import diff, c + +from pyramid.utils.array import diff, c, is_iterable from pyramid.utils import get_callable + from numpy.testing import assert_array_equal from nose.tools import assert_raises import numpy as np @@ -42,3 +44,10 @@ def test_corner_in_callable(): def test_corner(): # fails because lag < 1 assert_raises(ValueError, diff, x=x, lag=0) + + +def test_is_iterable(): + assert not is_iterable("this string") + assert is_iterable(["this", "list"]) + assert not is_iterable(None) + assert is_iterable(np.array([1, 2])) From e7f077bd70ef68d739849b56b6ac51c60a757dbc Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Fri, 5 Jan 2018 19:38:27 -0600 Subject: [PATCH 18/33] Refactor table initialization in ADFTest and PPTest to be static (avoid constant re-creation) --- pyramid/arima/stationarity.py | 72 +++++++++++++++++------------------ 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/pyramid/arima/stationarity.py b/pyramid/arima/stationarity.py index 22a7fc7f3..3e2564cc7 100644 --- a/pyramid/arima/stationarity.py +++ b/pyramid/arima/stationarity.py @@ -190,6 +190,18 @@ class ADFTest(_DifferencingStationarityTest): ---------- .. [1] https://wikipedia.org/wiki/Augmented_Dickey–Fuller_test """ + table = -np.array([c(4.38, 4.15, 4.04, 3.99, 3.98, 3.96), + c(3.95, 3.80, 3.73, 3.69, 3.68, 3.66), + c(3.60, 3.50, 3.45, 3.43, 3.42, 3.41), + c(3.24, 3.18, 3.15, 3.13, 3.13, 3.12), + c(1.14, 1.19, 1.22, 1.23, 1.24, 1.25), + c(0.80, 0.87, 0.90, 0.92, 0.93, 0.94), + c(0.50, 0.58, 0.62, 0.64, 0.65, 0.66), + c(0.15, 0.24, 0.28, 0.31, 0.32, 0.33)]).T + + tablen = table.shape[1] + tableT = c(25, 50, 100, 250, 500, 100000) + tablep = c(0.01, 0.025, 0.05, 0.10, 0.90, 0.95, 0.975, 0.99) def __init__(self, alpha=0.05, k=None): super(ADFTest, self).__init__(alpha=alpha) @@ -248,26 +260,14 @@ def is_stationary(self, x): # store them. res = sm.OLS(yt, X).fit() STAT = res.params[0] / res.HC0_se[0] # FIXME: is the denom correct?... - table = -np.array([c(4.38, 4.15, 4.04, 3.99, 3.98, 3.96), - c(3.95, 3.80, 3.73, 3.69, 3.68, 3.66), - c(3.60, 3.50, 3.45, 3.43, 3.42, 3.41), - c(3.24, 3.18, 3.15, 3.13, 3.13, 3.12), - c(1.14, 1.19, 1.22, 1.23, 1.24, 1.25), - c(0.80, 0.87, 0.90, 0.92, 0.93, 0.94), - c(0.50, 0.58, 0.62, 0.64, 0.65, 0.66), - c(0.15, 0.24, 0.28, 0.31, 0.32, 0.33)]).T - - tablen = table.shape[1] - tableT = c(25, 50, 100, 250, 500, 100000) - tablep = c(0.01, 0.025, 0.05, 0.10, 0.90, 0.95, 0.975, 0.99) - - tableipl = np.zeros(tablen) - for i in range(tablen): - _, pval = approx(tableT, table[:, i], xout=n, rule=2) + + tableipl = np.zeros(self.tablen) + for i in range(self.tablen): + _, pval = approx(self.tableT, self.table[:, i], xout=n, rule=2) tableipl[i] = pval # make sure to do 1 - x... - _, interpol = approx(tableipl, tablep, xout=STAT, rule=2) + _, interpol = approx(tableipl, self.tablep, xout=STAT, rule=2) pval = 1 - interpol[0] # in the R code, here is where the P value warning is tested again... @@ -300,6 +300,21 @@ class PPTest(_DifferencingStationarityTest): ---------- .. [1] R's tseries PP test source code: http://bit.ly/2wbzx6V """ + table = -np.array([ + c(22.5, 25.7, 27.4, 28.4, 28.9, 29.5), + c(19.9, 22.4, 23.6, 24.4, 24.8, 25.1), + c(17.9, 19.8, 20.7, 21.3, 21.5, 21.8), + c(15.6, 16.8, 17.5, 18.0, 18.1, 18.3), + c(3.66, 3.71, 3.74, 3.75, 3.76, 3.77), + c(2.51, 2.60, 2.62, 2.64, 2.65, 2.66), + c(1.53, 1.66, 1.73, 1.78, 1.78, 1.79), + c(0.43, 0.65, 0.75, 0.82, 0.84, 0.87) + ]).T + + tablen = table.shape[1] + tableT = c(25, 50, 100, 250, 500, 100000).astype(DTYPE) + tablep = c(0.01, 0.025, 0.05, 0.10, 0.90, 0.95, 0.975, 0.99) + def __init__(self, alpha=0.05, lshort=True): super(PPTest, self).__init__(alpha=alpha) @@ -365,28 +380,13 @@ def is_stationary(self, x): alpha = coef[2] # it's the last col... STAT = n * (alpha - 1) - (n ** 6) / (24.0 * dx) * (ssqrtl - ssqru) - table = -np.array([ - c(22.5, 25.7, 27.4, 28.4, 28.9, 29.5), - c(19.9, 22.4, 23.6, 24.4, 24.8, 25.1), - c(17.9, 19.8, 20.7, 21.3, 21.5, 21.8), - c(15.6, 16.8, 17.5, 18.0, 18.1, 18.3), - c(3.66, 3.71, 3.74, 3.75, 3.76, 3.77), - c(2.51, 2.60, 2.62, 2.64, 2.65, 2.66), - c(1.53, 1.66, 1.73, 1.78, 1.78, 1.79), - c(0.43, 0.65, 0.75, 0.82, 0.84, 0.87) - ]).T - - tablen = table.shape[1] - tableT = c(25, 50, 100, 250, 500, 100000).astype(DTYPE) - tablep = c(0.01, 0.025, 0.05, 0.10, 0.90, 0.95, 0.975, 0.99) - tableipl = np.zeros(tablen) - - for i in range(tablen): - _, pval = approx(tableT, table[:, i], xout=n, rule=2) + tableipl = np.zeros(self.tablen) + for i in range(self.tablen): + _, pval = approx(self.tableT, self.table[:, i], xout=n, rule=2) tableipl[i] = pval # make sure to do 1 - x... - _, interpol = approx(tableipl, tablep, xout=STAT, rule=2) + _, interpol = approx(tableipl, self.tablep, xout=STAT, rule=2) pval = 1 - interpol[0] # in the R code, here is where the P value warning is tested again... From 4e43186d2e1581fe0cee6c4d302a27f948fb28de Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Sun, 7 Jan 2018 07:53:13 -0600 Subject: [PATCH 19/33] Try rvm get stable for OS X build --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 74e0137f3..8a33008dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,9 @@ cache: - $HOME/.cache/pip - $HOME/.ccache -before_install: source build_tools/travis/before_install.sh +before_install: + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then rvm get stable; fi + - source build_tools/travis/before_install.sh env: global: - PYMODULE=pyramid From e076965cb523a77f9d3759ca6a86d34df7833f06 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Tue, 23 Jan 2018 08:58:22 -0600 Subject: [PATCH 20/33] Revert rvm get stable in .travis.yml --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8a33008dc..9ae330d6b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ cache: - $HOME/.ccache before_install: - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then rvm get stable; fi - source build_tools/travis/before_install.sh env: global: From 239551d5ff8e9af2d1ce3a01971b93d93abab12f Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Tue, 23 Jan 2018 12:30:33 -0600 Subject: [PATCH 21/33] Add missing lines to coveragerc, and add 'set +e' to .travis.yml --- .coveragerc | 3 +++ .travis.yml | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/.coveragerc b/.coveragerc index e2789456c..26ff2101a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,6 @@ +[report] +show_missing = True + [run] branch = True source = pyramid diff --git a/.travis.yml b/.travis.yml index 9ae330d6b..97b79ec85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,12 @@ cache: - $HOME/.cache/pip - $HOME/.ccache +# we set +e to re-instate the error-tolerant state. The problem is Travis is broken on Mac OS +# builds right now due to Issue 6307 (https://github.com/travis-ci/travis-ci/issues/6307) +# and fails at the end of a successful build. This will allow the build to recover even if +# a non-zero status code is encountered. before_install: + - set +e - source build_tools/travis/before_install.sh env: global: From d95509fa5f11dc056b0a2b625438356c293c89d6 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Wed, 24 Jan 2018 07:16:27 -0600 Subject: [PATCH 22/33] Update travis.yml with set +e --- .travis.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 97b79ec85..967a02143 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,12 +11,7 @@ cache: - $HOME/.cache/pip - $HOME/.ccache -# we set +e to re-instate the error-tolerant state. The problem is Travis is broken on Mac OS -# builds right now due to Issue 6307 (https://github.com/travis-ci/travis-ci/issues/6307) -# and fails at the end of a successful build. This will allow the build to recover even if -# a non-zero status code is encountered. before_install: - - set +e - source build_tools/travis/before_install.sh env: global: @@ -90,8 +85,14 @@ matrix: install: source build_tools/travis/install.sh before_script: bash build_tools/travis/before_script.sh script: bash build_tools/travis/test_script.sh + +# we set +e to re-instate the error-tolerant state. The problem is Travis is broken on Mac OS +# builds right now due to Issue 6307 (https://github.com/travis-ci/travis-ci/issues/6307) +# and fails at the end of a successful build. This will allow the build to recover even if +# a non-zero status code is encountered. after_success: - - source build_tools/travis/after_success.sh # || "shell_session_update failed" + - set +e + - source build_tools/travis/after_success.sh || echo "shell_session_update failed" # Build the wheels every time so we can debug - bash build_tools/travis/build_wheels.sh - ls dist/ From 65b74e8b5bd33ffecb800f23c95402da19e94a1b Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Wed, 24 Jan 2018 11:04:16 -0600 Subject: [PATCH 23/33] Update ruby for OSX build --- .travis.yml | 5 ++--- build_tools/travis/before_install.sh | 6 ++++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 967a02143..8f12b4d2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,10 +89,9 @@ script: bash build_tools/travis/test_script.sh # we set +e to re-instate the error-tolerant state. The problem is Travis is broken on Mac OS # builds right now due to Issue 6307 (https://github.com/travis-ci/travis-ci/issues/6307) # and fails at the end of a successful build. This will allow the build to recover even if -# a non-zero status code is encountered. +# a non-zero status code is encountered. (had `- set +e`) after_success: - - set +e - - source build_tools/travis/after_success.sh || echo "shell_session_update failed" + - source build_tools/travis/after_success.sh # || echo "shell_session_update failed" # Build the wheels every time so we can debug - bash build_tools/travis/build_wheels.sh - ls dist/ diff --git a/build_tools/travis/before_install.sh b/build_tools/travis/before_install.sh index e8cbf3a25..ab0b75563 100755 --- a/build_tools/travis/before_install.sh +++ b/build_tools/travis/before_install.sh @@ -7,6 +7,8 @@ if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then echo "Updating apt-get for Linux build" sudo apt-get -qq update # Because screw you, Travis: -# elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then -# rvm get stable +elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + echo "Updating Ruby for Mac OS build" + command curl -sSL https://rvm.io/mpapis.asc | gpg --import -; + rvm get stable fi From de3a78e66e2fb5ae2cd02b29a08f62ca8363c8c2 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Wed, 24 Jan 2018 15:07:30 -0600 Subject: [PATCH 24/33] Add custom ModelFitWarning and also hope Travis works... --- build_tools/travis/before_install.sh | 8 +++++++- pyramid/arima/__init__.py | 1 + pyramid/arima/auto.py | 4 +++- pyramid/arima/warnings.py | 18 ++++++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 pyramid/arima/warnings.py diff --git a/build_tools/travis/before_install.sh b/build_tools/travis/before_install.sh index ab0b75563..3663e4b84 100755 --- a/build_tools/travis/before_install.sh +++ b/build_tools/travis/before_install.sh @@ -6,7 +6,13 @@ set -e if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then echo "Updating apt-get for Linux build" sudo apt-get -qq update -# Because screw you, Travis: + +# Workaround for https://github.com/travis-ci/travis-ci/issues/6307, which +# caused the following error on MacOS X workers: +# +# Warning, RVM 1.26.0 introduces signed releases and automated check of +# signatures when GPG software found. +# /Users/travis/build.sh: line 109: shell_session_update: command not found elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then echo "Updating Ruby for Mac OS build" command curl -sSL https://rvm.io/mpapis.asc | gpg --import -; diff --git a/pyramid/arima/__init__.py b/pyramid/arima/__init__.py index 6da241ec0..c87d658aa 100644 --- a/pyramid/arima/__init__.py +++ b/pyramid/arima/__init__.py @@ -6,5 +6,6 @@ from .arima import * from .auto import * from .utils import * +from .warnings import * __all__ = [s for s in dir() if not s.startswith("_")] diff --git a/pyramid/arima/auto.py b/pyramid/arima/auto.py index 0e5d50cc3..749b0bf19 100644 --- a/pyramid/arima/auto.py +++ b/pyramid/arima/auto.py @@ -20,6 +20,7 @@ from .utils import ndiffs, is_constant, nsdiffs from ..utils import diff, is_iterable from .arima import ARIMA +from .warnings import ModelFitWarning # for python 3 compat from ..compat.python import xrange @@ -827,7 +828,8 @@ def _fit_arima(x, xreg, order, seasonal_order, start_params, trend, # for non-stationarity errors or singular matrices, return None except (LinAlgError, ValueError) as v: if error_action == 'warn': - warnings.warn(_fmt_warning_str(order, seasonal_order)) + warnings.warn(_fmt_warning_str(order, seasonal_order), + ModelFitWarning) elif error_action == 'raise': # todo: can we do something more informative in case # the error is not on the pyramid side? diff --git a/pyramid/arima/warnings.py b/pyramid/arima/warnings.py new file mode 100644 index 000000000..c8f00eecb --- /dev/null +++ b/pyramid/arima/warnings.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from __future__ import absolute_import + +__all__ = [ + 'ModelFitWarning' +] + + +class ModelFitWarning(UserWarning): + """Generic warning used for a model fit that might fail. More descriptive + than simply trying to lump everything into a default UserWarning, which + gives the user no insight into the reason for the warning apart from a + (potentially) cryptic message. This allows the user to understand the + warning emanates from an attempted model fit and originates from within + the Pyramid package. + """ + pass From 25898f53c649f4c4fb6d14793648c6322b3fae49 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Sat, 10 Feb 2018 16:55:47 -0600 Subject: [PATCH 25/33] Update before_install to remove rvm update and add echo to after_success --- .travis.yml | 2 +- build_tools/travis/after_success.sh | 5 +++++ build_tools/travis/before_install.sh | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8f12b4d2c..07b1da304 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,7 +91,7 @@ script: bash build_tools/travis/test_script.sh # and fails at the end of a successful build. This will allow the build to recover even if # a non-zero status code is encountered. (had `- set +e`) after_success: - - source build_tools/travis/after_success.sh # || echo "shell_session_update failed" + - source build_tools/travis/after_success.sh # || echo "shell_session_update failed" # Build the wheels every time so we can debug - bash build_tools/travis/build_wheels.sh - ls dist/ diff --git a/build_tools/travis/after_success.sh b/build_tools/travis/after_success.sh index cc2721541..f9d7b2942 100755 --- a/build_tools/travis/after_success.sh +++ b/build_tools/travis/after_success.sh @@ -8,6 +8,8 @@ set -e # push coverage if necessary if [[ "$COVERAGE" == "true" ]]; then + echo "COVERAGE=true; moving .coverage to build dir" + # Need to run coveralls from a git checkout, so we copy .coverage # from TEST_DIR where nosetests has been run cp $TEST_DIR/.coverage $TRAVIS_BUILD_DIR @@ -17,13 +19,16 @@ if [[ "$COVERAGE" == "true" ]]; then # very reliable but we don't want travis to report a failure # in the github UI just because the coverage report failed to # be published. + echo "Running coverage" coveralls || echo "Coveralls upload failed" fi # make sure we have twine in case we deploy +echo "Installing twine" pip install twine || "pip installing twine failed" # remove the .egg-info dir so Mac won't bomb on bdist_wheel cmd (absolute path in SOURCES.txt) +echo "Removing egg info (if exists)" rm -r pyramid_arima.egg-info/ || echo "No local .egg cache to remove" # make a dist folder if not there, then make sure permissions are sufficient diff --git a/build_tools/travis/before_install.sh b/build_tools/travis/before_install.sh index 3663e4b84..f75ab0d6f 100755 --- a/build_tools/travis/before_install.sh +++ b/build_tools/travis/before_install.sh @@ -15,6 +15,6 @@ if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then # /Users/travis/build.sh: line 109: shell_session_update: command not found elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then echo "Updating Ruby for Mac OS build" - command curl -sSL https://rvm.io/mpapis.asc | gpg --import -; - rvm get stable + # command curl -sSL https://rvm.io/mpapis.asc | gpg --import -; + # rvm get stable fi From 2d61bea2ebc6fdb8e3f7823715d0964486dc4dcf Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Sat, 10 Feb 2018 17:05:57 -0600 Subject: [PATCH 26/33] Can we use rsync instead of cp? --- build_tools/travis/after_success.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_tools/travis/after_success.sh b/build_tools/travis/after_success.sh index f9d7b2942..346d4d1e8 100755 --- a/build_tools/travis/after_success.sh +++ b/build_tools/travis/after_success.sh @@ -12,7 +12,7 @@ if [[ "$COVERAGE" == "true" ]]; then # Need to run coveralls from a git checkout, so we copy .coverage # from TEST_DIR where nosetests has been run - cp $TEST_DIR/.coverage $TRAVIS_BUILD_DIR + rsync $TEST_DIR/.coverage $TRAVIS_BUILD_DIR --ignore-existing cd $TRAVIS_BUILD_DIR # Ignore coveralls failures as the coveralls server is not From 075a1c375eee34c850266962c7b3e2da403c012f Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Sat, 10 Feb 2018 17:17:18 -0600 Subject: [PATCH 27/33] Remove -e from OSX failing travis script --- build_tools/travis/after_success.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_tools/travis/after_success.sh b/build_tools/travis/after_success.sh index 346d4d1e8..9cbfd547c 100755 --- a/build_tools/travis/after_success.sh +++ b/build_tools/travis/after_success.sh @@ -4,7 +4,8 @@ # License: 3-clause BSD -set -e +# 02/10/2018 remove due to Travis build issue 6307 +# set -e # push coverage if necessary if [[ "$COVERAGE" == "true" ]]; then From f6daab58993e028923b191073b4ae3ed9293442c Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Sat, 10 Feb 2018 17:23:47 -0600 Subject: [PATCH 28/33] Seriously, Travis sucks. --- build_tools/travis/after_success.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build_tools/travis/after_success.sh b/build_tools/travis/after_success.sh index 9cbfd547c..de55b62f2 100755 --- a/build_tools/travis/after_success.sh +++ b/build_tools/travis/after_success.sh @@ -6,6 +6,7 @@ # 02/10/2018 remove due to Travis build issue 6307 # set -e +set +e # because TRAVIS SUCKS # push coverage if necessary if [[ "$COVERAGE" == "true" ]]; then From 0034cd92c02f5ca3d25f4b7b91cd3ba185319ec2 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Sat, 10 Feb 2018 17:37:34 -0600 Subject: [PATCH 29/33] Bump version --- pyramid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyramid/__init__.py b/pyramid/__init__.py index 608565b38..bc73e1217 100644 --- a/pyramid/__init__.py +++ b/pyramid/__init__.py @@ -4,7 +4,7 @@ # # The pyramid module -__version__ = "0.6.5-dev" +__version__ = "0.6.5" try: # this var is injected in the setup build to enable From 89695f5059cd241875458b15fdc030ad4d4f408f Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Sat, 10 Feb 2018 17:38:07 -0600 Subject: [PATCH 30/33] [ci skip] Bump version --- pyramid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyramid/__init__.py b/pyramid/__init__.py index 608565b38..3f6876ef0 100644 --- a/pyramid/__init__.py +++ b/pyramid/__init__.py @@ -4,7 +4,7 @@ # # The pyramid module -__version__ = "0.6.5-dev" +__version__ = "0.7.0-dev" try: # this var is injected in the setup build to enable From b2092d9ae5b931974ca6915633684162e8799d07 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Mon, 12 Feb 2018 07:21:01 -0600 Subject: [PATCH 31/33] Not sure if rsync is preventing .coverage from making it into the coveralls dir --- build_tools/travis/after_success.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/build_tools/travis/after_success.sh b/build_tools/travis/after_success.sh index de55b62f2..0fd93ddaf 100755 --- a/build_tools/travis/after_success.sh +++ b/build_tools/travis/after_success.sh @@ -10,27 +10,23 @@ set +e # because TRAVIS SUCKS # push coverage if necessary if [[ "$COVERAGE" == "true" ]]; then - echo "COVERAGE=true; moving .coverage to build dir" # Need to run coveralls from a git checkout, so we copy .coverage # from TEST_DIR where nosetests has been run - rsync $TEST_DIR/.coverage $TRAVIS_BUILD_DIR --ignore-existing + cp $TEST_DIR/.coverage $TRAVIS_BUILD_DIR cd $TRAVIS_BUILD_DIR # Ignore coveralls failures as the coveralls server is not # very reliable but we don't want travis to report a failure # in the github UI just because the coverage report failed to # be published. - echo "Running coverage" coveralls || echo "Coveralls upload failed" fi # make sure we have twine in case we deploy -echo "Installing twine" pip install twine || "pip installing twine failed" # remove the .egg-info dir so Mac won't bomb on bdist_wheel cmd (absolute path in SOURCES.txt) -echo "Removing egg info (if exists)" rm -r pyramid_arima.egg-info/ || echo "No local .egg cache to remove" # make a dist folder if not there, then make sure permissions are sufficient From c0fd56dd48a37c26ba5262ae090668f82968c497 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Mon, 12 Feb 2018 08:05:33 -0600 Subject: [PATCH 32/33] Revert .coveragerc to previous version--coverage seems to be suffering --- .coveragerc | 5 +---- pyramid/utils/array.py | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.coveragerc b/.coveragerc index 26ff2101a..c412019f4 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,3 @@ -[report] -show_missing = True - [run] branch = True source = pyramid @@ -8,4 +5,4 @@ include = */pyramid/* omit = */setup.py */__check_build/* - */_build_utils/* + */_build_utils/* \ No newline at end of file diff --git a/pyramid/utils/array.py b/pyramid/utils/array.py index c58f26a87..9ae4cc9c5 100644 --- a/pyramid/utils/array.py +++ b/pyramid/utils/array.py @@ -64,7 +64,28 @@ def c(*args): # otherwise it's not iterable, put it in an array return np.asarray([element]) - # concat all + # np.concat all. This can be slow, as noted by numerous threads on + # numpy concat efficiency, however an alternative using recursive + # yields was tested and performed far worse: + # + # >>> def timeit(func, ntimes, *args): + # ... times = [] + # ... for i in range(ntimes): + # ... start = time.time() + # ... func(*args) + # ... times.append(time.time() - start) + # ... arr = np.asarray(times) + # ... print("%s (%i times) - Mean: %.5f sec, " + # ... "Min: %.5f sec, Max: %.5f" % (func.__name__, ntimes, + # ... arr.mean(), arr.min(), + # ... arr.max())) + # >>> y = [np.arange(10000), range(500), (1000,), 100, np.arange(50000)] + # >>> timeit(c1, 100, *y) + # c1 (100 times) - Mean: 0.00009 sec, Min: 0.00006 sec, Max: 0.00065 + # >>> timeit(c2, 100, *y) + # c2 (100 times) - Mean: 0.08708 sec, Min: 0.08273 sec, Max: 0.10115 + # + # So we stick with c1, which is this variant. return np.concatenate([a if is_iterable(a) else [a] for a in args]) From a882f6e304f335a41a53c827868476b321a49417 Mon Sep 17 00:00:00 2001 From: Taylor Smith Date: Mon, 12 Feb 2018 08:06:16 -0600 Subject: [PATCH 33/33] Fix version --- pyramid/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyramid/__init__.py b/pyramid/__init__.py index 3f6876ef0..bc73e1217 100644 --- a/pyramid/__init__.py +++ b/pyramid/__init__.py @@ -4,7 +4,7 @@ # # The pyramid module -__version__ = "0.7.0-dev" +__version__ = "0.6.5" try: # this var is injected in the setup build to enable