Merge pull request statsmodels#5587 from bashtage/additional-deprecat…

…ions

MAINT: Resolve additional deprecations

setup.cfg

@@ -12,6 +12,7 @@ filterwarnings =
     ignore:Using a non-tuple:FutureWarning:scipy.signal
     ignore:Using a non-tuple:FutureWarning:scipy.stats.stats
     ignore:the matrix subclass:PendingDeprecationWarning:numpy.matrixlib.defmatrix
+    ignore:Method .ptp is deprecated:FutureWarning:numpy.core.fromnumeric
 markers =
     example: mark a test that runs example code
     matplotlib: mark a test that requires matplotlib

statsmodels/discrete/tests/test_discrete.py

@@ -482,7 +482,7 @@ def setup_class(cls):
         cls.res1 = Probit(data.endog, data.exog).fit(method="minimize", disp=0,
                                                      maxiter=500,
                                                      min_method='Nelder-Mead',
-                                                     xtol=1e-4, ftol=1e-4)
+                                                     xatol=1e-4, fatol=1e-4)
 
 class CheckLikelihoodModelL1(object):
     """
@@ -1440,7 +1440,7 @@ def test_perfect_prediction():
     iris_dir = os.path.join(cur_dir, '..', '..', 'genmod', 'tests', 'results')
     iris_dir = os.path.abspath(iris_dir)
     iris = np.genfromtxt(os.path.join(iris_dir, 'iris.csv'), delimiter=",",
-                            skip_header=1)
+                         skip_header=1)
     y = iris[:,-1]
     X = iris[:,:-1]
     X = X[y != 2]

statsmodels/gam/generalized_additive_model.py

@@ -9,7 +9,10 @@
 """
 
 from __future__ import division
-import collections
+try:
+    from collections.abc import Iterable
+except ImportError:  # Python 2.7
+    from collections import Iterable
 import copy  # check if needed when dropping python 2.7
 
 import numpy as np
@@ -598,7 +601,7 @@ def _check_alpha(self, alpha):
             smooth terms
 
         """
-        if not isinstance(alpha, collections.Iterable):
+        if not isinstance(alpha, Iterable):
             alpha = [alpha] * len(self.smoother.smoothers)
         elif not isinstance(alpha, list):
             # we want alpha to be a list
@@ -935,7 +938,7 @@ class LogitGam(PenalizedMixin, Logit):
 
     """
     def __init__(self, endog, smoother, alpha, *args, **kwargs):
-        if not isinstance(alpha, collections.Iterable):
+        if not isinstance(alpha, Iterable):
             alpha = np.array([alpha] * len(smoother.smoothers))
 
         self.smoother = smoother

statsmodels/nonparametric/tests/test_lowess.py

@@ -78,7 +78,7 @@ def test_delta_0(self):
 
     def test_delta_rdef(self):
         self.generate('test_delta_Rdef', 'test_lowess_delta.csv', out='out_Rdef',
-                      kwargs=lambda data: {'frac': .1, 'delta': .01 * data['x'].ptp()})
+                      kwargs=lambda data: {'frac': .1, 'delta': .01 * np.ptp(data['x'])})
 
     def test_delta_1(self):
         self.generate('test_delta_1', 'test_lowess_delta.csv', out='out_1',

statsmodels/sandbox/panel/correlation_structures.py

@@ -76,7 +76,7 @@ def corr_arma(k_vars, ar, ma):
     from scipy.linalg import toeplitz
     from statsmodels.tsa.arima_process import arma2ar
 
-    ar = arma2ar(ar, ma, nobs=k_vars)[:k_vars]  #bug in arma2ar
+    ar = arma2ar(ar, ma, lags=k_vars)[:k_vars]  #bug in arma2ar
 
     return toeplitz(ar)
 

statsmodels/sandbox/predict_functional.py

@@ -190,7 +190,7 @@ def _make_exog_from_formula(result, focus_var, summaries, values, num_points):
     for ky in values.keys():
         fexog.loc[:, ky] = values[ky]
 
-    dexog = patsy.dmatrix(model.data.design_info.builder, fexog, return_type='dataframe')
+    dexog = patsy.dmatrix(model.data.design_info, fexog, return_type='dataframe')
     return dexog, fexog, fvals
 
 
@@ -303,13 +303,13 @@ def predict_functional(result, focus_var, summaries=None, values=None,
             raise ValueError("if `exog` is provided then do not provide `summaries` or `values`")
 
         fexog = exog
-        dexog = patsy.dmatrix(model.data.design_info.builder,
+        dexog = patsy.dmatrix(model.data.design_info,
                               fexog, return_type='dataframe')
         fvals = exog[focus_var]
 
         if exog2 is not None:
             fexog2 = exog
-            dexog2 = patsy.dmatrix(model.data.design_info.builder,
+            dexog2 = patsy.dmatrix(model.data.design_info,
                                    fexog2, return_type='dataframe')
             fvals2 = fvals
 

statsmodels/stats/tests/test_nonparametric.py

@@ -8,9 +8,13 @@
 from statsmodels.compat.python import lzip, range
 import numpy as np
 from numpy.testing import assert_allclose, assert_almost_equal
-from statsmodels.sandbox.stats.runs import (mcnemar, cochrans_q, Runs,
-                                            symmetry_bowker,
+import pytest
+
+from statsmodels.stats.contingency_tables import mcnemar, cochrans_q, SquareTable
+from statsmodels.sandbox.stats.runs import (Runs, symmetry_bowker,
                                             runstest_1samp, runstest_2samp)
+from statsmodels.sandbox.stats.runs import mcnemar as sbmcnemar
+
 
 def _expand_table(table):
     '''expand a 2 by 2 contingency table to observations
@@ -33,17 +37,18 @@ def test_mcnemar_exact():
     res4 = 0.00206
     res5 = 0.002221
     res6 = 1.
-
-    assert_almost_equal(mcnemar(f_obs1, exact=True), [59, res1], decimal=6)
-    assert_almost_equal(mcnemar(f_obs2, exact=True), [59, res2], decimal=6)
-    assert_almost_equal(mcnemar(f_obs3, exact=True), [59, res3], decimal=6)
-    assert_almost_equal(mcnemar(f_obs4, exact=True), [30, res4], decimal=6)
-    assert_almost_equal(mcnemar(f_obs5, exact=True), [10, res5], decimal=6)
-    assert_almost_equal(mcnemar(f_obs6, exact=True), [10, res6], decimal=6)
-
-    x, y = _expand_table(f_obs2).T  # tuple unpack
-    assert_allclose(mcnemar(f_obs2, exact=True),
-                    mcnemar(x, y, exact=True), rtol=1e-13)
+    stat = mcnemar(f_obs1, exact=True)
+    assert_almost_equal([stat.statistic, stat.pvalue], [59, res1], decimal=6)
+    stat = mcnemar(f_obs2, exact=True)
+    assert_almost_equal([stat.statistic, stat.pvalue], [59, res2], decimal=6)
+    stat = mcnemar(f_obs3, exact=True)
+    assert_almost_equal([stat.statistic, stat.pvalue], [59, res3], decimal=6)
+    stat = mcnemar(f_obs4, exact=True)
+    assert_almost_equal([stat.statistic, stat.pvalue], [30, res4], decimal=6)
+    stat = mcnemar(f_obs5, exact=True)
+    assert_almost_equal([stat.statistic, stat.pvalue], [10, res5], decimal=6)
+    stat = mcnemar(f_obs6, exact=True)
+    assert_almost_equal([stat.statistic, stat.pvalue], [10, res6], decimal=6)
 
 
 def test_mcnemar_chisquare():
@@ -56,75 +61,81 @@ def test_mcnemar_chisquare():
     res2 = [0.7751938,    0.3786151]
     res3 = [2.87769784,   0.08981434]
 
-    assert_allclose(mcnemar(f_obs1, exact=False), res1, rtol=1e-6)
-    assert_allclose(mcnemar(f_obs2, exact=False), res2, rtol=1e-6)
-    assert_allclose(mcnemar(f_obs3, exact=False), res3, rtol=1e-6)
-
-    # compare table versus observations
-    x, y = _expand_table(f_obs2).T  # tuple unpack
-    assert_allclose(mcnemar(f_obs2, exact=False),
-                    mcnemar(x, y, exact=False), rtol=1e-13)
+    stat = mcnemar(f_obs1, exact=False)
+    assert_allclose([stat.statistic, stat.pvalue], res1, rtol=1e-6)
+    stat = mcnemar(f_obs2, exact=False)
+    assert_allclose([stat.statistic, stat.pvalue], res2, rtol=1e-6)
+    stat = mcnemar(f_obs3, exact=False)
+    assert_allclose([stat.statistic, stat.pvalue], res3, rtol=1e-6)
 
     # test correction = False
     res1 = [2.135556e01, 3.815136e-06]
     res2 = [0.9379845,   0.3327967]
     res3 = [3.17266187,  0.07488031]
 
     res = mcnemar(f_obs1, exact=False, correction=False)
-    assert_allclose(res, res1, rtol=1e-6)
+    assert_allclose([res.statistic, res.pvalue], res1, rtol=1e-6)
     res = mcnemar(f_obs2, exact=False, correction=False)
-    assert_allclose(res, res2, rtol=1e-6)
+    assert_allclose([res.statistic, res.pvalue], res2, rtol=1e-6)
     res = mcnemar(f_obs3, exact=False, correction=False)
-    assert_allclose(res, res3, rtol=1e-6)
+    assert_allclose([res.statistic, res.pvalue], res3, rtol=1e-6)
 
 
 def test_mcnemar_vectorized(reset_randomstate):
     ttk = np.random.randint(5,15, size=(2,2,3))
-    mcnemar(ttk)
-    res = mcnemar(ttk, exact=False)
-    res1 = lzip(*[mcnemar(ttk[:,:,i], exact=False) for i in range(3)])
+    with pytest.deprecated_call():
+        res = sbmcnemar(ttk, exact=False)
+    with pytest.deprecated_call():
+        res1 = lzip(*[sbmcnemar(ttk[:, :, i], exact=False) for i in range(3)])
     assert_allclose(res, res1, rtol=1e-13)
 
-    res = mcnemar(ttk, exact=False, correction=False)
-    res1 = lzip(*[mcnemar(ttk[:,:,i], exact=False, correction=False)
-                                                          for i in range(3)])
+    with pytest.deprecated_call():
+        res = sbmcnemar(ttk, exact=False, correction=False)
+    with pytest.deprecated_call():
+        res1 = lzip(*[sbmcnemar(ttk[:, :, i], exact=False, correction=False)
+                      for i in range(3)])
     assert_allclose(res, res1, rtol=1e-13)
 
-    res = mcnemar(ttk, exact=True)
-    res1 = lzip(*[mcnemar(ttk[:,:,i], exact=True) for i in range(3)])
+    with pytest.deprecated_call():
+        res = sbmcnemar(ttk, exact=True)
+    with pytest.deprecated_call():
+        res1 = lzip(*[sbmcnemar(ttk[:, :, i], exact=True) for i in range(3)])
     assert_allclose(res, res1, rtol=1e-13)
 
 
 def test_symmetry_bowker():
     table = np.array([0, 3, 4, 4, 2, 4, 1, 2, 4, 3, 5, 3, 0, 0, 2, 2, 3, 0, 0,
                       1, 5, 5, 5, 5, 5]).reshape(5, 5)
 
-    res = symmetry_bowker(table)
+    res = SquareTable(table, shift_zeros=False).symmetry()
     mcnemar5_1 = dict(statistic=7.001587, pvalue=0.7252951, parameters=(10,),
                       distr='chi2')
-    assert_allclose(res[:2], [mcnemar5_1['statistic'], mcnemar5_1['pvalue']],
+    assert_allclose([res.statistic, res.pvalue],
+                    [mcnemar5_1['statistic'], mcnemar5_1['pvalue']],
                     rtol=1e-7)
 
-    res = symmetry_bowker(1 + table)
+    res = SquareTable(1 + table, shift_zeros=False).symmetry()
     mcnemar5_1b = dict(statistic=5.355988, pvalue=0.8661652, parameters=(10,),
                        distr='chi2')
-    assert_allclose(res[:2], [mcnemar5_1b['statistic'], mcnemar5_1b['pvalue']],
+    assert_allclose([res.statistic, res.pvalue],
+                    [mcnemar5_1b['statistic'], mcnemar5_1b['pvalue']],
                     rtol=1e-7)
 
-
     table = np.array([2, 2, 3, 6, 2, 3, 4, 3, 6, 6, 6, 7, 1, 9, 6, 7, 1, 1, 9,
                       8, 0, 1, 8, 9, 4]).reshape(5, 5)
 
-    res = symmetry_bowker(table)
+    res = SquareTable(table, shift_zeros=False).symmetry()
     mcnemar5_2 = dict(statistic=18.76432, pvalue=0.04336035, parameters=(10,),
                       distr='chi2')
-    assert_allclose(res[:2], [mcnemar5_2['statistic'], mcnemar5_2['pvalue']],
+    assert_allclose([res.statistic, res.pvalue],
+                    [mcnemar5_2['statistic'], mcnemar5_2['pvalue']],
                     rtol=1.5e-7)
 
-    res = symmetry_bowker(1 + table)
+    res = SquareTable(1 + table, shift_zeros=False).symmetry()
     mcnemar5_2b = dict(statistic=14.55256, pvalue=0.1492461, parameters=(10,),
                        distr='chi2')
-    assert_allclose(res[:2], [mcnemar5_2b['statistic'], mcnemar5_2b['pvalue']],
+    assert_allclose([res.statistic, res.pvalue],
+                    [mcnemar5_2b['statistic'], mcnemar5_2b['pvalue']],
                     rtol=1e-7)
 
 
@@ -145,12 +156,15 @@ def test_cochransq():
                    [1, 1, 1]])
     res_qstat = 2.8
     res_pvalue = 0.246597
-    assert_almost_equal(cochrans_q(x), [res_qstat, res_pvalue])
+    res = cochrans_q(x)
+    assert_almost_equal([res.statistic, res.pvalue], [res_qstat, res_pvalue])
 
     #equivalence of mcnemar and cochranq for 2 samples
     a,b = x[:,:2].T
-    assert_almost_equal(mcnemar(a,b, exact=False, correction=False),
-                        cochrans_q(x[:,:2]))
+    res = cochrans_q(x[:, :2])
+    with pytest.deprecated_call():
+        assert_almost_equal(sbmcnemar(a, b, exact=False, correction=False),
+                            [res.statistic, res.pvalue])
 
 
 def test_cochransq2():
@@ -170,7 +184,7 @@ def test_cochransq2():
         0 0 1 1'''.split(), int).reshape(-1, 4)
 
     res = cochrans_q(data)
-    assert_allclose(res, [13.2857143, 0.00405776], rtol=1e-6)
+    assert_allclose([res.statistic, res.pvalue], [13.2857143, 0.00405776], rtol=1e-6)
 
 
 def test_cochransq3():
@@ -198,7 +212,7 @@ def test_cochransq3():
     data = np.repeat(cases, count, 0)
 
     res = cochrans_q(data)
-    assert_allclose(res, [8.4706, 0.0145], atol=5e-5)
+    assert_allclose([res.statistic, res.pvalue], [8.4706, 0.0145], atol=5e-5)
 
 def test_runstest(reset_randomstate):
     #comparison numbers from R, tseries, runs.test

statsmodels/stats/tests/test_pairwise.py

@@ -5,6 +5,7 @@
 
 Author: Josef Perktold
 """
+from statsmodels.compat.numpy import NP_LT_114
 from statsmodels.compat.python import BytesIO, asbytes, range
 
 import warnings
@@ -115,7 +116,8 @@
 3 - 2\t-4.340\t-7.989\t-0.691\t***
 3 - 1\t0.260\t-3.389\t3.909\t-
 1 - 2\t-4.600\t-8.249\t-0.951\t***
-1 - 3\t-0.260\t-3.909\t3.389\t'''
+1 - 3\t-0.260\t-3.909\t3.389\t-
+'''
 
 cylinders = np.array([8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 4, 8, 8, 8, 8, 8, 8, 8, 8, 8, 4, 6, 6, 6, 4, 4,
                     4, 4, 4, 4, 6, 8, 8, 8, 8, 4, 4, 4, 4, 8, 8, 8, 8, 6, 6, 6, 6, 4, 4, 4, 4, 6, 6,
@@ -136,15 +138,17 @@
 ss3 = asbytes(ss3)
 ss5 = asbytes(ss5)
 
-dta = np.recfromtxt(BytesIO(ss), names=("Rust", "Brand", "Replication"))
-dta2 = np.recfromtxt(BytesIO(ss2), names=("idx", "Treatment", "StressReduction"))
-dta3 = np.recfromtxt(BytesIO(ss3), names=("Brand", "Relief"))
-dta5 = np.recfromtxt(BytesIO(ss5), names=('pair', 'mean', 'lower', 'upper', 'sig'), delimiter='\t')
-
-dta = pd.DataFrame.from_records(dta)
-dta2 = pd.DataFrame.from_records(dta2)
-dta3 = pd.DataFrame.from_records(dta3)
-dta5 = pd.DataFrame.from_records(dta5)
+dta = pd.read_csv(BytesIO(ss), sep=r'\s+', header=None)
+dta.columns = "Rust", "Brand", "Replication"
+dta2 = pd.read_csv(BytesIO(ss2), sep=r'\s+', header=None)
+dta2.columns = "idx", "Treatment", "StressReduction"
+dta2["Treatment"] = dta2["Treatment"].map(lambda v: v.encode('utf-8'))
+dta3 = pd.read_csv(BytesIO(ss3), sep=r'\s+', header=None)
+dta3.columns = ["Brand", "Relief"]
+dta5 = pd.read_csv(BytesIO(ss5), sep=r'\t', header=None)
+dta5.columns = ['pair', 'mean', 'lower', 'upper', 'sig']
+for col in ('pair', 'sig'):
+    dta5[col] = dta5[col].map(lambda v: v.encode('utf-8'))
 sas_ = dta5.iloc[[1, 3, 2]]
 
 from statsmodels.stats.multicomp import (tukeyhsd, pairwise_tukeyhsd,

statsmodels/tools/grouputils.py

@@ -366,19 +366,13 @@ def levels(self):
     @property
     def labels(self):
         # this was index_int, but that's not a very good name...
-        if hasattr(self.index, 'labels'):
-            return self.index.labels
-        else:  # pandas version issue here
-            # Compat code for the labels -> codes change in pandas 0.15
-            # FIXME: use .codes directly when we don't want to support
-            # pandas < 0.15
-            tmp = pd.Categorical(self.index)
-            try:
-                labl = tmp.codes
-            except AttributeError:
-                labl = tmp.labels  # Old pandsd
-
-            return labl[None]
+        codes = getattr(self.index, 'codes', None)
+        if codes is None:
+            if hasattr(self.index, 'labels'):
+                codes = self.index.labels
+            else:
+                codes = pd.Categorical(self.index).codes[None]
+        return codes
 
     @property
     def group_names(self):