Skip to content

Commit

Permalink
Merge pull request #3652 from pycaret/fix_ts_deprecations
Browse files Browse the repository at this point in the history 
Fix for sktime changes in recent versions
  • Loading branch information
@ngupta23
ngupta23 committed Jul 24, 2023
2 parents c4cb387 + c71d77c commit 6662100
Show file tree
Hide file tree
Showing 4 changed files with 177 additions and 127 deletions.
147 changes: 79 additions & 68 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,49 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.7, 3.8, 3.9, "3.10"]
python-version: [3.8, 3.9, "3.10"]

steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
# SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL required due to
# pyLDAvis 3.3.1 having an "sklearn" dependency
run: |
export SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL=True
python -m pip install -U pip
python -m pip install -U pytest numpy
pip install ".[full, test]"
if [ -f requirements-prophet.txt ]; then pip install -r requirements-prophet.txt; fi
- name: Python version and dependency list
run: |
echo "Python version expected: ${{ matrix.python-version }}"
python --version
which python
pip list
- name: Remove tests
run: |
rm tests/test_classification_tuning.py
rm tests/test_classification_plots.py
rm tests/test_regression_plots.py
rm tests/test_regression_tuning.py
rm tests/test_time_series_tune_grid.py
rm tests/test_time_series_tune_random.py
rm tests/test_time_series_plots.py
rm tests/test_time_series_utils_plots.py
rm tests/benchmarks/*.py
- name: Test with pytest
run: pytest --durations=0

test_linux_3p7:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.7]

steps:
- uses: actions/checkout@v3
Expand Down Expand Up @@ -106,7 +148,42 @@ jobs:
runs-on: windows-latest
strategy:
matrix:
python-version: [3.7, 3.8, 3.9, "3.10"]
python-version: [3.8, 3.9, "3.10"]

steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
# SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL required due to
# pyLDAvis 3.3.1 having an "sklearn" dependency
run: |
$env:SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL = 'True'
python -m pip install -U pip
python -m pip install -U pytest numpy
pip install ".[full, test]"
pip install -r requirements-prophet.txt
- name: Python version and dependency list
run: |
echo "Python version expected: ${{ matrix.python-version }}"
python --version
which python
pip list
- name: Remove tests
run: |
remove-item tests/* -Include @('test_clustering.py', 'test_classification_tuning.py','test_classification_plots.py','test_regression_plots.py', 'test_regression_tuning.py', 'test_time_series_tune_grid.py', 'test_time_series_tune_random.py', 'test_create_api.py', 'test_create_docker.py', 'test_drift_report.py', 'test_eda.py', 'test_time_series_plots.py', 'test_time_series_utils_plots.py')
remove-item tests/benchmarks/*
- name: Test with pytest
run: pytest --durations=0

test_windows_3p7:
runs-on: windows-latest
strategy:
matrix:
python-version: [3.7]

steps:
- uses: actions/checkout@v3
Expand Down Expand Up @@ -268,69 +345,3 @@ jobs:
- name: Run benchmarks
run: pytest tests/benchmarks --durations=0

# test_tuning:

# runs-on: ubuntu-latest

# steps:
# - uses: actions/checkout@v3
# - name: Set up Python 3.8
# uses: actions/setup-python@v4
# with:
# python-version: 3.8
# - name: Install dependencies
# run: |
# python -m pip install --upgrade pip
# python -m pip install -U pytest
# python -m pip install codecov
# pip install ".[full]"
# python -m pip install hpbandster ConfigSpace
# - name: Remove tests
# run: |
# find tests -type f -not -name '__init__.py' -not -name 'test_classification_tuning.py' -not -name 'test_regression_tuning.py' -delete
# - name: Test with pytest
# run: pytest --durations=0

# test_tuning_clf_windows:

# runs-on: windows-latest

# steps:
# - uses: actions/checkout@v3
# - name: Set up Python 3.8
# uses: actions/setup-python@v4
# with:
# python-version: 3.8
# - name: Install dependencies
# run: |
# python -m pip install --upgrade pip
# python -m pip install -U pytest
# python -m pip install codecov
# pip install ".[full]"
# - name: Remove tests
# run: |
# remove-item tests/* -Exclude @('__init__.py','test_classification_tuning.py')
# - name: Test with pytest
# run: pytest --durations=0

# test_tuning_reg_windows:

# runs-on: windows-latest

# steps:
# - uses: actions/checkout@v3
# - name: Set up Python 3.8
# uses: actions/setup-python@v4
# with:
# python-version: 3.8
# - name: Install dependencies
# run: |
# python -m pip install --upgrade pip
# python -m pip install -U pytest
# python -m pip install codecov
# pip install ".[full]"
# - name: Remove tests
# run: |
# remove-item tests/* -Exclude @('__init__.py','test_regression_tuning.py')
# - name: Test with pytest
# run: pytest --durations=0
18 changes: 16 additions & 2 deletions pycaret/containers/models/time_series.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import random
import warnings
from abc import abstractmethod
from inspect import getfullargspec
from typing import Any, Dict, List, Optional, Tuple, Union

import numpy as np # type: ignore
Expand Down Expand Up @@ -2752,7 +2753,12 @@ def predict(self, fh=None, X=None):

return y

def predict_quantiles(self, fh, X=None, alpha=None):
# From sktime
# https://github.com/sktime/sktime/blob/c20afabc576385259e2e0d37f46bc67da0928f16/sktime/forecasting/base/_base.py#L2085C5-L2086C48
# https://github.com/sktime/sktime/compare/v0.20.1...v0.21.0#diff-ef86237172bb2f91c585e845b38d6fb28bb6ac128ff8efa541cdd86f81751bf2
# todo 0.22.0 - switch legacy_interface default to False
# todo 0.23.0 - remove legacy_interface arg
def predict_quantiles(self, fh, X=None, alpha=None, legacy_interface=True):
"""Compute/return prediction quantiles for a forecast.
private _predict_quantiles containing the core logic,
Expand Down Expand Up @@ -2796,7 +2802,15 @@ def predict_quantiles(self, fh, X=None, alpha=None):
# Hence coerce the index internally if it is not DatetimeIndex
X = coerce_period_to_datetime_index(X)

preds = super().predict_quantiles(fh=fh, X=X, alpha=alpha)
# https://github.com/sktime/sktime/blob/94c6355802bf91239aef36e71c56808410d6eb3f/sktime/forecasting/base/_base.py#L590
has_li_arg = (
"legacy_interface" in getfullargspec(super().predict_quantiles).args
)
if has_li_arg:
kwargs = {"legacy_interface": legacy_interface}
else:
kwargs = {}
preds = super().predict_quantiles(fh=fh, X=X, alpha=alpha, **kwargs)

# sktime Prophet returns back DatetimeIndex
# Convert back to PeriodIndex for pycaret
Expand Down
5 changes: 4 additions & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ plotly-resampler>=0.8.3.1
statsmodels>=0.12.1
# Can not be ==0.17.1 due to this bug in 0.17.1: https://github.com/sktime/sktime/issues/4468
# Can not be >=0.17.2 due to this bug (fixed in 0.18.1): https://github.com/sktime/sktime/issues/4587
sktime>=0.16.1,!=0.17.1,!=0.17.2,!=0.18.0
# Can't >=0.22.0 because of legacy_interface default will switch to False in 0.22.0
# This restriction can be removed if the new interface is handled in pycaret for newer versions
# of sktime and the older interface is handled in pycaret for older versions of sktime
sktime>=0.16.1,!=0.17.1,!=0.17.2,!=0.18.0,<0.22.0
tbats>=1.1.3
pmdarima>=1.8.0,!=1.8.1,<3.0.0 # Matches sktime
134 changes: 78 additions & 56 deletions tests/test_time_series_preprocess.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

import numpy as np
import pytest
import sktime
from packaging import version
from sktime.forecasting.compose import ForecastingPipeline, TransformedTargetForecaster
from time_series_test_utils import (
_IMPUTE_METHODS_STR,
Expand Down Expand Up @@ -206,73 +208,79 @@ def test_preprocess_setup_raises_missing_exo(load_uni_exo_data_target_missing):
def test_preprocess_setup_raises_negative_no_exo(load_pos_and_neg_data, method):
"""Tests setup conditions that raise errors due to negative values before
transformatons. Univariate without exogenous variables"""
data = load_pos_and_neg_data

exp = TSForecastingExperiment()

with pytest.raises(ValueError) as errmsg:
exp.setup(data=data, transform_target=method)
exceptionmsg = errmsg.value.args[0]
# The first message is given when then the transformation produced NA values
# and the underlying model can handle missing data (in this case, the sktime
# checks pass but pycaret checks fail)
# The second message is given the transformation produces NA values and the
# underlying model can not handle missing data (in this case, the sktime checks
# fail and the pycaret checks are not reached.)
assert (
"This can happen when you have negative and/or zero values in the data"
or "DummyForecaster cannot handle missing data (nans), but y passed contained missing data"
in exceptionmsg
)
continue_ = _continue_negative_value_checks(method=method)
if continue_:
data = load_pos_and_neg_data

exp = TSForecastingExperiment()

with pytest.raises(ValueError) as errmsg:
exp.setup(data=data, transform_target=method)
exceptionmsg = errmsg.value.args[0]
# The first message is given when then the transformation produced NA values
# and the underlying model can handle missing data (in this case, the sktime
# checks pass but pycaret checks fail)
# The second message is given the transformation produces NA values and the
# underlying model can not handle missing data (in this case, the sktime checks
# fail and the pycaret checks are not reached.)
assert (
"This can happen when you have negative and/or zero values in the data"
or "DummyForecaster cannot handle missing data (nans), but y passed contained missing data"
in exceptionmsg
)


@pytest.mark.parametrize("method", _TRANSFORMATION_METHODS_NO_NEG)
def test_preprocess_setup_raises_negative_exo(load_uni_exo_data_target, method):
"""Tests setup conditions that raise errors due to negative values before
transformations. Univariate with exogenous variables"""
data, target = load_uni_exo_data_target

exp = TSForecastingExperiment()

# Transform Target ----
with pytest.raises(ValueError) as errmsg:
exp.setup(
data=data,
target=target,
transform_target=method,
continue_ = _continue_negative_value_checks(method=method)
if continue_:
data, target = load_uni_exo_data_target

exp = TSForecastingExperiment()

# Transform Target ----
with pytest.raises(ValueError) as errmsg:
exp.setup(
data=data,
target=target,
transform_target=method,
)
exceptionmsg = errmsg.value.args[0]
# The first message is given when then the transformation produced NA values
# and the underlying model can handle missing data (in this case, the sktime
# checks pass but pycaret checks fail)
# The second message is given the transformation produces NA values and the
# underlying model can not handle missing data (in this case, the sktime checks
# fail and the pycaret checks are not reached.)
assert (
"This can happen when you have negative and/or zero values in the data"
or "DummyForecaster cannot handle missing data (nans), but y passed contained missing data"
in exceptionmsg
)
exceptionmsg = errmsg.value.args[0]
# The first message is given when then the transformation produced NA values
# and the underlying model can handle missing data (in this case, the sktime
# checks pass but pycaret checks fail)
# The second message is given the transformation produces NA values and the
# underlying model can not handle missing data (in this case, the sktime checks
# fail and the pycaret checks are not reached.)
assert (
"This can happen when you have negative and/or zero values in the data"
or "DummyForecaster cannot handle missing data (nans), but y passed contained missing data"
in exceptionmsg
)

# Transform Exogenous ----
with pytest.raises(ValueError) as errmsg:
exp.setup(
data=data,
target=target,
transform_exogenous=method,
# Transform Exogenous ----
with pytest.raises(ValueError) as errmsg:
exp.setup(
data=data,
target=target,
transform_exogenous=method,
)
exceptionmsg = errmsg.value.args[0]
# The first message is given when then the transformation produced NA values
# and the underlying model can handle missing data (in this case, the sktime
# checks pass but pycaret checks fail)
# The second message is given the transformation produces NA values and the
# underlying model can not handle missing data (in this case, the sktime checks
# fail and the pycaret checks are not reached.)
assert (
"This can happen when you have negative and/or zero values in the data"
or "DummyForecaster cannot handle missing data (nans), but y passed contained missing data"
in exceptionmsg
)
exceptionmsg = errmsg.value.args[0]
# The first message is given when then the transformation produced NA values
# and the underlying model can handle missing data (in this case, the sktime
# checks pass but pycaret checks fail)
# The second message is given the transformation produces NA values and the
# underlying model can not handle missing data (in this case, the sktime checks
# fail and the pycaret checks are not reached.)
assert (
"This can happen when you have negative and/or zero values in the data"
or "DummyForecaster cannot handle missing data (nans), but y passed contained missing data"
in exceptionmsg
)


@pytest.mark.parametrize("model_name", _model_names_for_missing_data)
Expand Down Expand Up @@ -751,3 +759,17 @@ def test_no_transform_exo(load_uni_exo_data_target_missing):
assert missing_imputed_data_test.iloc[0].equals(
missing_imputed_data_all_train.iloc[0]
)


def _continue_negative_value_checks(method):
"""Checks if the negative value checks should be continued"""
continue_ = True

# Negative values are handled in Boc Cox Transformer after sktime 0.20.1
# https://github.com/sktime/sktime/pull/4770
if method == "box-cox" and version.parse(sktime.__version__) >= version.parse(
"0.20.1"
):
continue_ = False

return continue_

0 comments on commit 6662100

Please sign in to comment.