CLN: Fixes for future SciPY and pandas

Silence new warnings produced by pandas 1.0+
Ensure starting values respect builds in HoltWinters

lint.sh

@@ -17,122 +17,215 @@ if [ "$LINT" == true ]; then
     # pass _all_ flake8 checks
     echo "Linting known clean files with strict rules"
     flake8 --isolated \
+        examples/ \
+        setup.py \
         statsmodels/__init__.py \
+        statsmodels/_version.py \
+        statsmodels/api.py \
+        statsmodels/base/__init__.py \
         statsmodels/base/distributed_estimation.py \
         statsmodels/base/elastic_net.py \
         statsmodels/base/tests/test_data.py \
         statsmodels/base/tests/test_distributed_estimation.py \
+        statsmodels/base/tests/test_shrink_pickle.py \
         statsmodels/base/tests/test_transform.py \
         statsmodels/compat/ \
         statsmodels/conftest.py \
-        statsmodels/datasets/elec_equip \
+        statsmodels/datasets/__init__.py \
+        statsmodels/datasets/anes96/__init__.py \
+        statsmodels/datasets/cancer/__init__.py \
+        statsmodels/datasets/ccard/__init__.py \
+        statsmodels/datasets/china_smoking/__init__.py \
+        statsmodels/datasets/co2/__init__.py \
+        statsmodels/datasets/committee/__init__.py \
+        statsmodels/datasets/copper/__init__.py \
+        statsmodels/datasets/cpunish/__init__.py \
+        statsmodels/datasets/elec_equip/ \
+        statsmodels/datasets/elnino/__init__.py \
+        statsmodels/datasets/engel/__init__.py \
+        statsmodels/datasets/fair/__init__.py \
+        statsmodels/datasets/fertility/__init__.py \
+        statsmodels/datasets/grunfeld/__init__.py \
+        statsmodels/datasets/heart/__init__.py \
+        statsmodels/datasets/interest_inflation/__init__.py \
+        statsmodels/datasets/longley/__init__.py \
+        statsmodels/datasets/macrodata/__init__.py \
+        statsmodels/datasets/modechoice/__init__.py \
+        statsmodels/datasets/nile/__init__.py \
+        statsmodels/datasets/randhie/__init__.py \
+        statsmodels/datasets/scotland/__init__.py \
+        statsmodels/datasets/spector/__init__.py \
+        statsmodels/datasets/stackloss/__init__.py \
+        statsmodels/datasets/star98/__init__.py \
+        statsmodels/datasets/statecrime/__init__.py \
+        statsmodels/datasets/strikes/__init__.py \
+        statsmodels/datasets/sunspots/__init__.py \
         statsmodels/datasets/tests/ \
+        statsmodels/discrete/__init__.py \
         statsmodels/discrete/tests/results/ \
         statsmodels/discrete/tests/test_constrained.py \
+        statsmodels/distributions/__init__.py \
         statsmodels/duration/__init__.py \
+        statsmodels/duration/_kernel_estimates.py \
+        statsmodels/duration/api.py \
         statsmodels/duration/tests/results/ \
+        statsmodels/duration/tests/test_survfunc.py \
+        statsmodels/emplike/__init__.py \
+        statsmodels/emplike/api.py \
         statsmodels/emplike/tests/ \
         statsmodels/examples/tests/ \
         statsmodels/formula/ \
         statsmodels/gam/ \
+        statsmodels/genmod/__init__.py \
         statsmodels/genmod/_tweedie_compound_poisson.py \
+        statsmodels/genmod/api.py \
         statsmodels/genmod/bayes_mixed_glm.py \
         statsmodels/genmod/families/ \
         statsmodels/genmod/generalized_estimating_equations.py \
         statsmodels/genmod/qif.py \
         statsmodels/genmod/tests/results/ \
         statsmodels/genmod/tests/test_gee.py \
         statsmodels/genmod/tests/test_qif.py \
+        statsmodels/graphics/__init__.py \
+        statsmodels/graphics/api.py \
+        statsmodels/graphics/functional.py \
+        statsmodels/graphics/tests/test_agreement.py \
+        statsmodels/graphics/tests/test_correlation.py \
+        statsmodels/graphics/tests/test_functional.py \
+        statsmodels/graphics/tests/test_gofplots.py \
         statsmodels/graphics/tsaplots.py \
+        statsmodels/imputation/__init__.py \
         statsmodels/interface/ \
+        statsmodels/iolib/__init__.py \
+        statsmodels/iolib/api.py \
         statsmodels/iolib/openfile.py \
         statsmodels/iolib/smpickle.py \
         statsmodels/iolib/summary2.py \
         statsmodels/iolib/table.py \
         statsmodels/iolib/tableformatting.py \
         statsmodels/iolib/tests/results/ \
         statsmodels/iolib/tests/test_pickle.py \
+        statsmodels/miscmodels/__init__.py \
+        statsmodels/miscmodels/tests/test_tarma.py \
+        statsmodels/multivariate/__init__.py \
+        statsmodels/multivariate/api.py \
+        statsmodels/multivariate/factor_rotation/_analytic_rotation.py \
         statsmodels/multivariate/pca.py \
         statsmodels/multivariate/tests/results/ \
+        statsmodels/nonparametric/__init__.py \
+        statsmodels/nonparametric/api.py \
+        statsmodels/nonparametric/tests/results/ \
+        statsmodels/regression/__init__.py \
+        statsmodels/regression/_prediction.py \
+        statsmodels/regression/_tools.py \
         statsmodels/regression/dimred.py \
         statsmodels/regression/mixed_linear_model.py \
         statsmodels/regression/process_regression.py \
         statsmodels/regression/recursive_ls.py \
+        statsmodels/regression/rolling.py \
         statsmodels/regression/tests/results/ \
         statsmodels/regression/tests/test_dimred.py \
         statsmodels/regression/tests/test_lme.py \
         statsmodels/regression/tests/test_processreg.py \
         statsmodels/regression/tests/test_quantile_regression.py \
+        statsmodels/regression/tests/test_rolling.py \
         statsmodels/regression/tests/test_tools.py \
-        statsmodels/regression/tests/results/ \
-        statsmodels/robust/norms.py \
-        statsmodels/robust/scale.py \
         statsmodels/resampling/ \
-        statsmodels/robust/tests/ \
+        statsmodels/robust/ \
+        statsmodels/sandbox/__init__.py \
+        statsmodels/sandbox/distributions/__init__.py \
         statsmodels/sandbox/distributions/tests/test_gof_new.py \
         statsmodels/sandbox/distributions/try_pot.py \
+        statsmodels/sandbox/nonparametric/__init__.py \
         statsmodels/sandbox/panel/correlation_structures.py \
         statsmodels/sandbox/regression/tests/results_gmm_griliches.py \
         statsmodels/sandbox/regression/tests/results_gmm_griliches_iter.py \
         statsmodels/sandbox/regression/tests/results_gmm_poisson.py \
         statsmodels/sandbox/regression/tests/results_ivreg2_griliches.py \
+        statsmodels/sandbox/stats/__init__.py \
         statsmodels/sandbox/stats/ex_multicomp.py \
-        statsmodels/sandbox/stats/tests/test_multicomp.py \
+        statsmodels/sandbox/stats/tests/ \
+        statsmodels/src/ \
         statsmodels/stats/__init__.py \
         statsmodels/stats/_knockoff.py \
+        statsmodels/stats/_lilliefors.py \
+        statsmodels/stats/_lilliefors_critical_values.py \
+        statsmodels/stats/api.py \
         statsmodels/stats/base.py \
+        statsmodels/stats/contingency_tables.py \
         statsmodels/stats/correlation_tools.py \
+        statsmodels/stats/diagnostic.py \
+        statsmodels/stats/dist_dependence_measures.py \
         statsmodels/stats/knockoff_regeffects.py \
+        statsmodels/stats/libqsturng/__init__.py \
         statsmodels/stats/moment_helpers.py \
         statsmodels/stats/multicomp.py \
+        statsmodels/stats/oaxaca.py \
         statsmodels/stats/regularized_covariance.py \
         statsmodels/stats/stattools.py \
-        statsmodels/stats/oaxaca.py \
+        statsmodels/stats/tabledist.py \
         statsmodels/stats/tests/results/ \
         statsmodels/stats/tests/test_anova_rm.py \
         statsmodels/stats/tests/test_correlation.py \
         statsmodels/stats/tests/test_descriptivestats.py \
+        statsmodels/stats/tests/test_dist_dependant_measures.py \
         statsmodels/stats/tests/test_knockoff.py \
         statsmodels/stats/tests/test_lilliefors.py \
         statsmodels/stats/tests/test_moment_helpers.py \
         statsmodels/stats/tests/test_multi.py \
+        statsmodels/stats/tests/test_oaxaca.py \
+        statsmodels/stats/tests/test_outliers_influence.py \
         statsmodels/stats/tests/test_qsturng.py \
         statsmodels/stats/tests/test_regularized_covariance.py \
-        statsmodels/stats/tests/test_oaxaca.py \
+        statsmodels/stats/tests/test_tabledist.py \
         statsmodels/tests/ \
         statsmodels/tools/decorators.py \
         statsmodels/tools/docstring.py \
         statsmodels/tools/linalg.py \
         statsmodels/tools/sm_exceptions.py \
         statsmodels/tools/tests/test_decorators.py \
+        statsmodels/tools/tests/test_docstring.py \
         statsmodels/tools/tests/test_linalg.py \
         statsmodels/tools/validation/ \
         statsmodels/tools/web.py \
-        statsmodels/tools/tests/test_docstring.py \
+        statsmodels/tsa/__init__.py \
+        statsmodels/tsa/_bds.py \
         statsmodels/tsa/adfvalues.py \
+        statsmodels/tsa/api.py \
         statsmodels/tsa/ar_model.py \
         statsmodels/tsa/arima/ \
         statsmodels/tsa/arima_model.py \
+        statsmodels/tsa/base/__init__.py \
         statsmodels/tsa/base/tests/test_datetools.py \
+        statsmodels/tsa/exponential_smoothing/ \
+        statsmodels/tsa/filters/__init__.py \
+        statsmodels/tsa/filters/api.py \
+        statsmodels/tsa/filters/bk_filter.py \
+        statsmodels/tsa/filters/hp_filter.py \
         statsmodels/tsa/filters/tests/ \
         statsmodels/tsa/innovations/ \
+        statsmodels/tsa/interp/__init__.py \
         statsmodels/tsa/kalmanf/ \
         statsmodels/tsa/regime_switching/ \
         statsmodels/tsa/seasonal.py \
         statsmodels/tsa/statespace/ \
         statsmodels/tsa/tests/results/ \
         statsmodels/tsa/tests/test_ar.py \
         statsmodels/tsa/tests/test_arima.py \
+        statsmodels/tsa/tests/test_bds.py \
         statsmodels/tsa/tests/test_seasonal.py \
         statsmodels/tsa/tests/test_stl.py \
         statsmodels/tsa/tests/test_x13.py \
+        statsmodels/tsa/vector_ar/__init__.py \
+        statsmodels/tsa/vector_ar/api.py \
         statsmodels/tsa/vector_ar/dynamic.py \
+        statsmodels/tsa/vector_ar/hypothesis_test_results.py \
+        statsmodels/tsa/vector_ar/tests/JMulTi_results/ \
+        statsmodels/tsa/vector_ar/tests/Matlab_results/ \
         statsmodels/tsa/vector_ar/tests/results/ \
         statsmodels/tsa/x13.py \
-        examples/ \
-        tools/ \
-        setup.py
-    if [ $? -ne "0" ]; then
+        tools/
+        if [ $? -ne "0" ]; then
         echo "Previously passing files failed linting."
         RET=1
     fi

statsmodels/base/data.py

@@ -587,7 +587,7 @@ def attach_dates(self, result):
         squeezed = result.squeeze()
         k_endog = np.array(self.ynames, ndmin=1).shape[0]
         if k_endog > 1 and squeezed.shape == (k_endog,):
-            squeezed = squeezed[None, :]
+            squeezed = np.asarray(squeezed)[None, :]
         # May be zero-dim, for example in the case of forecast one step in tsa
         if squeezed.ndim < 2:
             return Series(squeezed, index=self.predict_dates)

statsmodels/base/tests/test_generic_methods.py

@@ -13,6 +13,7 @@
 from statsmodels.compat.pandas import assert_series_equal, assert_index_equal
 from statsmodels.compat.platform import (PLATFORM_OSX, PLATFORM_LINUX32,
                                          PLATFORM_WIN32)
+from statsmodels.compat.scipy import SCIPY_GT_14
 
 import numpy as np
 import pandas as pd
@@ -114,7 +115,7 @@ def test_zero_constrained(self):
         assert_allclose(tvals1, res2.tvalues, rtol=tol, atol=tol)
 
         # See gh5993
-        if PLATFORM_LINUX32:
+        if PLATFORM_LINUX32 or SCIPY_GT_14:
             pvals1 = res1.pvalues[keep_index_p]
         else:
             with pytest.warns(RuntimeWarning,
@@ -264,7 +265,7 @@ def test_zero_collinear(self):
             assert_allclose(tvals1, res2.tvalues, rtol=5e-8)
 
             # See gh5993
-            if PLATFORM_LINUX32:
+            if PLATFORM_LINUX32 or SCIPY_GT_14:
                 pvals1 = res1.pvalues[keep_index_p]
             else:
                 with pytest.warns(RuntimeWarning,

statsmodels/compat/scipy.py

@@ -6,6 +6,8 @@
 SCIPY_11 = (LooseVersion(scipy.__version__) < LooseVersion('1.2.0') and
             LooseVersion(scipy.__version__) >= LooseVersion('1.1.0'))
 
+SCIPY_GT_14 = LooseVersion(scipy.__version__) >= LooseVersion('1.5')
+
 
 def _next_regular(target):
     """

statsmodels/graphics/regressionplots.py

@@ -406,6 +406,8 @@ def plot_partregress(endog, exog_i, exog_others, data=None,
     # all arrays or pandas-like
 
     if RHS_isemtpy:
+        endog = np.asarray(endog)
+        exog_i = np.asarray(exog_i)
         ax.plot(endog, exog_i, 'o', **kwargs)
         fitted_line = OLS(endog, exog_i).fit()
         x_axis_endog_name = 'x' if isinstance(exog_i, np.ndarray) else exog_i.name
@@ -428,11 +430,11 @@ def plot_partregress(endog, exog_i, exog_others, data=None,
     ax.set_ylabel("e(%s | X)" % y_axis_endog_name)
     ax.set_title('Partial Regression Plot', **title_kwargs)
 
-    #NOTE: if we want to get super fancy, we could annotate if a point is
-    #clicked using this widget
-    #http://stackoverflow.com/questions/4652439/
-    #is-there-a-matplotlib-equivalent-of-matlabs-datacursormode/
-    #4674445#4674445
+    # NOTE: if we want to get super fancy, we could annotate if a point is
+    # clicked using this widget
+    # http://stackoverflow.com/questions/4652439/
+    # is-there-a-matplotlib-equivalent-of-matlabs-datacursormode/
+    # 4674445#4674445
     if obs_labels is True:
         if data is not None:
             obs_labels = data.index

statsmodels/graphics/tests/test_regressionplots.py

@@ -140,9 +140,9 @@ class TestPlotFormula(TestPlotPandas):
     def test_one_column_exog(self, close_figures):
         from statsmodels.formula.api import ols
         res = ols("y~var1-1", data=self.data).fit()
-        fig = plot_regress_exog(res, "var1")
+        plot_regress_exog(res, "var1")
         res = ols("y~var1", data=self.data).fit()
-        fig = plot_regress_exog(res, "var1")
+        plot_regress_exog(res, "var1")
 
 
 class TestABLine(object):

statsmodels/sandbox/predict_functional.py

@@ -421,7 +421,7 @@ def _glm_basic_scr(result, exog, alpha):
 
     # The variance and SD of the linear predictor at each row of exog.
     sigma2 = (np.dot(exog, cov) * exog).sum(1)
-    sigma = np.sqrt(sigma2)
+    sigma = np.asarray(np.sqrt(sigma2))
 
     # Calculate kappa_0 (formula 42 from Sun et al)
     bz = np.linalg.solve(B.T, exog.T).T

statsmodels/tsa/holtwinters.py

@@ -680,12 +680,13 @@ def fit(self, smoothing_level=None, smoothing_slope=None, smoothing_seasonal=Non
                 # using guesstimates for the levels
                 txi = xi & np.array([True, True, True, False, False, True] + [False] * m)
                 txi = txi.astype(np.bool)
-                bounds = np.array([(0.0, 1.0), (0.0, 1.0), (0.0, 1.0),
-                                   (0.0, None), (0.0, None), (0.0, 1.0)] + [(None, None), ] * m)
+                bounds = ([(0.0, 1.0), (0.0, 1.0), (0.0, 1.0), (0.0, None),
+                           (0.0, None), (0.0, 1.0)] + [(None, None), ] * m)
                 args = (txi.astype(np.uint8), p, y, lvls, b, s, m, self.nobs,
                         max_seen)
                 if start_params is None and np.any(txi) and use_brute:
-                    res = brute(func, bounds[txi], args, Ns=20,
+                    _bounds = [bnd for bnd, flag in zip(bounds, txi) if flag]
+                    res = brute(func, _bounds, args, Ns=20,
                                 full_output=True, finish=None)
                     p[txi], max_seen, _, _ = res
                 else:
@@ -708,14 +709,23 @@ def fit(self, smoothing_level=None, smoothing_slope=None, smoothing_seasonal=Non
                     # Take a deeper look in the local minimum we are in to find the best
                     # solution to parameters, maybe hop around to try escape the local
                     # minimum we may be in.
+                    _bounds = [bnd for bnd, flag in zip(bounds, xi) if flag]
                     res = basinhopping(func, p[xi],
-                                       minimizer_kwargs={'args': args, 'bounds': bounds[xi]},
+                                       minimizer_kwargs={'args': args, 'bounds': _bounds},
                                        stepsize=0.01)
                     success = res.lowest_optimization_result.success
                 else:
                     # Take a deeper look in the local minimum we are in to find the best
                     # solution to parameters
-                    res = minimize(func, p[xi], args=args, bounds=bounds[xi])
+                    _bounds = [bnd for bnd, flag in zip(bounds, xi) if flag]
+                    lb, ub = np.asarray(_bounds).T.astype(np.float)
+                    initial_p = p[xi]
+                    loc = p[xi] < lb
+                    eps = np.finfo(np.double).eps
+                    initial_p[loc] = lb[loc] + eps * (ub[loc] - lb[loc])
+                    loc = p[xi] > ub
+                    initial_p[loc] = ub[loc] - eps * (ub[loc] - lb[loc])
+                    res = minimize(func, p[xi], args=args, bounds=_bounds)
                     success = res.success
 
                 if not success:

statsmodels/tsa/tests/test_holtwinters.py

@@ -323,8 +323,6 @@ def test_holt_damp_R(self):
             447.2614880558126]
         assert_allclose(fit.forecast(10), desired, atol=1e-4)
 
-
-
     def test_hw_seasonal(self):
         fit1 = ExponentialSmoothing(self.aust, seasonal_periods=4,
                                     trend='additive',