-
Notifications
You must be signed in to change notification settings - Fork 813
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feature/multivariate - step 4 #111
Changes from 184 commits
a1ea75b
7ccc04e
ec0c496
65f3ceb
d046bdd
13b9afe
f66e7be
b4a3322
4116ef3
fee47e8
77568b6
9aa93ff
d397e5b
500a38c
5fbb44c
e91a508
844017c
03560e2
80cc905
b71571f
f2838b2
6580bd2
0bcb174
27c599e
4e1cdd2
b7a5797
2db6474
776a439
1dfdea8
8da1df5
7648a51
5d5f1c8
8212bba
336758c
2546145
7247562
2ccf35f
81b6018
224df58
1681591
7ad1d01
67ead17
e71b1b8
a4937cf
2a2603d
25f0545
a985cb6
c01bea7
ef00405
f97a3f2
04aa1c5
a26eac1
b3445a4
2152306
df81f12
f3006c0
5713095
6e6fcc0
b3ff9c5
fa2674d
1677001
342610a
9e71d84
fe98f40
9b2eef5
7ac40fa
d7ac73c
7d1c8e1
0fe7a39
54dc266
a61bfc9
6981403
1ae7dc8
d3043ff
a8b6d7a
130a8c0
86f7948
49dd263
517a987
cbf5e3b
78378d0
ac1ab6e
738a316
9fbfe11
894b8bb
be63e27
e2fc0db
b75d80f
0fb18f0
86aaa3b
cf9e2d0
4a9876f
422f1a8
be78170
756aebb
fe771b4
bf2f75b
14385f0
a92ec7a
f559426
29952c9
72ed3eb
7d9cbb7
41fada9
e538d92
6668bc4
0eaffbf
663b7d7
67d209c
88f1ef8
59d20dc
e658ab2
6c45ad2
cdb9cbf
e8e2232
d810d9d
8d1b638
033eea4
ac47b52
15a0293
74f3957
7e42775
9d14112
b8d5f28
8c9cbf0
87dfb7c
a547ff4
667285a
0ac4103
2511083
0997bdf
5e6299b
78f965d
4bc025c
4fac6a4
a97d7fa
effdecd
8198b56
cbcebe4
b9a5bcb
1e5c4d8
7eb54e5
904e404
148566c
749b3d2
648aeeb
58b688a
9cbae0c
698eb0c
9c238f6
c0c3955
d05d94b
fb9f944
917408d
8c80ad8
05c9abc
74d3081
7da6efc
4a1d9d4
199b50b
6dacd70
2cb6d53
4f54a8d
ec6f01c
a3f1025
03a5157
57029af
9d629af
1722873
81ab93f
57bcc6e
8abbf6c
3e5931b
f1d6e54
bab99e0
9ecbe30
044840b
d4ea330
5143028
b2d99a2
332415d
c6141ef
6274acc
5804e41
22526ad
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,7 +3,7 @@ | |
--------------------- | ||
""" | ||
|
||
from typing import Iterable, Optional, Callable | ||
from typing import Iterable, Optional, Callable, List | ||
from itertools import product | ||
import math | ||
import time | ||
|
@@ -13,7 +13,8 @@ | |
import matplotlib.pyplot as plt | ||
|
||
from ..timeseries import TimeSeries | ||
from ..models.forecasting_model import ForecastingModel | ||
from ..models.forecasting_model import ForecastingModel, UnivariateForecastingModel | ||
from ..models.torch_forecasting_model import TorchForecastingModel | ||
from ..models.regression_model import RegressionModel | ||
from ..models import NaiveSeasonal, AutoARIMA, ExponentialSmoothing, FFT, Prophet, Theta | ||
from .. import metrics | ||
|
@@ -28,10 +29,28 @@ | |
|
||
# TODO parameterize the moving window | ||
|
||
def _create_parameter_dicts(model, target_indices, component_index, use_full_output_length): | ||
fit_kwargs = {} | ||
predict_kwargs = {} | ||
if isinstance(model, UnivariateForecastingModel): | ||
fit_kwargs['component_index'] = component_index | ||
else: | ||
fit_kwargs['target_indices'] = target_indices | ||
if isinstance(model, TorchForecastingModel): | ||
predict_kwargs['use_full_output_length'] = use_full_output_length | ||
|
||
return fit_kwargs, predict_kwargs | ||
|
||
|
||
def backtest_forecasting(series: TimeSeries, | ||
model: ForecastingModel, | ||
start: pd.Timestamp, | ||
fcast_horizon_n: int, | ||
target_indices: Optional[List[int]] = None, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Quite a lot of args passed to this function. As an optional improvement can consider moving them to a class level and wrap the whole backtesting module in a class. |
||
component_index: Optional[int] = None, | ||
use_full_output_length: bool = True, | ||
stride: int = 1, | ||
retrain: bool = True, | ||
trim_to_series: bool = True, | ||
verbose: bool = False) -> TimeSeries: | ||
""" A function for backtesting `ForecastingModel`'s. | ||
|
@@ -45,9 +64,14 @@ def backtest_forecasting(series: TimeSeries, | |
forecast horizon, and then moves the end of the training set forward by one | ||
time step. The resulting predictions are then returned. | ||
|
||
This always re-trains the models on the entire available history, | ||
Unless `retrain` is set to False, this always re-trains the models on the entire available history, | ||
corresponding an expending window strategy. | ||
|
||
If `retrain` is set to False (useful for models with many parameter such as `TorchForecastingModel` instances), | ||
the model will only be trained only on the initial training window (up to `start` time stamp), | ||
and only if it has not been trained before. Then, at every iteration, the newly expanded 'training sequence' | ||
will be fed to the model to produce the new output. | ||
|
||
Parameters | ||
---------- | ||
series | ||
|
@@ -58,6 +82,20 @@ def backtest_forecasting(series: TimeSeries, | |
The first prediction time, at which a prediction is computed for a future time | ||
fcast_horizon_n | ||
The forecast horizon for the point predictions | ||
target_indices | ||
In case `series` is multivariate and `model` is a subclass of `MultivariateForecastingModel`, | ||
a list of indices of components of `series` to be predicted by `model`. | ||
component_index | ||
In case `series` is multivariate and `model` is a subclass of `UnivariateForecastingModel`, | ||
an integer index of the component of `series` to be predicted by `model`. | ||
use_full_output_length | ||
In case `model` is a subclass of `TorchForecastingModel`, this argument will be passed along | ||
as argument to the predict method of `model`. | ||
stride | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. very nice. It would also be nice to have the option of configuring the training set length. When set, this would do moving window, and when not set it would do expending window (can wait a future PR though :) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes for sure, @guillaumeraille also suggested this! |
||
The number of time steps (the unit being the frequency of `series`) between two consecutive predictions. | ||
retrain | ||
Whether to retrain the model for every prediction or not. Currently only `TorchForecastingModel` | ||
instances as `model` argument support setting `retrain` to `False`. | ||
trim_to_series | ||
Whether the predicted series has the end trimmed to match the end of the main series | ||
verbose | ||
|
@@ -70,30 +108,42 @@ def backtest_forecasting(series: TimeSeries, | |
the specified model with the specified forecast horizon. | ||
""" | ||
|
||
series._assert_univariate() | ||
raise_if_not(start in series, 'The provided start timestamp is not in the time series.', logger) | ||
raise_if_not(start != series.end_time(), 'The provided start timestamp is the last timestamp of the time series', | ||
logger) | ||
raise_if_not(fcast_horizon_n > 0, 'The provided forecasting horizon must be a positive integer.', logger) | ||
raise_if_not(retrain or isinstance(model, TorchForecastingModel), "Only 'TorchForecastingModel' instances" | ||
" support the option 'retrain=False'.", logger) | ||
|
||
last_pred_time = series.time_index()[-fcast_horizon_n - 1] if trim_to_series else series.time_index()[-1] | ||
last_pred_time = ( | ||
series.time_index()[-fcast_horizon_n - stride] if trim_to_series else series.time_index()[-stride - 1] | ||
) | ||
|
||
# specify the correct fit and predict keyword arguments for the given model | ||
fit_kwargs, predict_kwargs = _create_parameter_dicts(model, target_indices, component_index, use_full_output_length) | ||
|
||
# build the prediction times in advance (to be able to use tqdm) | ||
pred_times = [start] | ||
while pred_times[-1] <= last_pred_time: | ||
pred_times.append(pred_times[-1] + series.freq()) | ||
pred_times.append(pred_times[-1] + series.freq() * stride) | ||
|
||
# what we'll return | ||
values = [] | ||
times = [] | ||
|
||
iterator = _build_tqdm_iterator(pred_times, verbose) | ||
|
||
if ((not retrain) and (not model._fit_called)): | ||
model.fit(series.drop_after(start), verbose=verbose, **fit_kwargs) | ||
|
||
for pred_time in iterator: | ||
train = series.drop_after(pred_time) # build the training series | ||
model.fit(train) | ||
pred = model.predict(fcast_horizon_n) | ||
values.append(pred.univariate_values()[-1]) # store the N-th point | ||
if (retrain): | ||
model.fit(train, **fit_kwargs) | ||
pred = model.predict(fcast_horizon_n, **predict_kwargs) | ||
else: | ||
pred = model.predict(fcast_horizon_n, input_series=train, **predict_kwargs) | ||
values.append(pred.values()[-1]) # store the N-th point | ||
times.append(pred.end_time()) # store the N-th timestamp | ||
return TimeSeries.from_times_and_values(pd.DatetimeIndex(times), np.array(values)) | ||
|
||
|
@@ -144,7 +194,6 @@ def backtest_regression(feature_series: Iterable[TimeSeries], | |
the specified model with the specified forecast horizon. | ||
""" | ||
|
||
raise_if_not(target_series.width == 1, "'target_series' must be univariate.", logger) | ||
raise_if_not(all([s.has_same_time_as(target_series) for s in feature_series]), 'All provided time series must ' | ||
'have the same time index', logger) | ||
raise_if_not(start in target_series, 'The provided start timestamp is not in the time series.', logger) | ||
|
@@ -174,7 +223,7 @@ def backtest_regression(feature_series: Iterable[TimeSeries], | |
|
||
model.fit(train_features, train_target) | ||
pred = model.predict(val_features) | ||
values.append(pred.univariate_values()[-1]) # store the N-th point | ||
values.append(pred.values()[-1]) # store the N-th point | ||
times.append(pred.end_time()) # store the N-th timestamp | ||
|
||
return TimeSeries.from_times_and_values(pd.DatetimeIndex(times), np.array(values)) | ||
|
@@ -183,7 +232,7 @@ def backtest_regression(feature_series: Iterable[TimeSeries], | |
def forecasting_residuals(model: ForecastingModel, | ||
series: TimeSeries, | ||
fcast_horizon_n: int = 1, | ||
verbose: bool = True) -> TimeSeries: | ||
verbose: bool = False) -> TimeSeries: | ||
""" A function for computing the residuals produced by a given model and univariate time series. | ||
|
||
This function computes the difference between the actual observations from `series` | ||
|
@@ -285,6 +334,9 @@ def backtest_gridsearch(model_class: type, | |
parameters: dict, | ||
train_series: TimeSeries, | ||
fcast_horizon_n: Optional[int] = None, | ||
target_indices: Optional[List[int]] = None, | ||
component_index: Optional[int] = None, | ||
use_full_output_length: bool = True, | ||
val_series: Optional[TimeSeries] = None, | ||
num_predictions: int = 10, | ||
metric: Callable[[TimeSeries, TimeSeries], float] = metrics.mape, | ||
|
@@ -324,6 +376,15 @@ def backtest_gridsearch(model_class: type, | |
The univariate TimeSeries instance used for validation in split mode. | ||
fcast_horizon_n | ||
The integer value of the forecasting horizon used in expanding window mode. | ||
target_indices | ||
In case `series` is multivariate and `model` is a subclass of `MultivariateForecastingModel`, | ||
a list of indices of components of `series` to be predicted by `model`. | ||
component_index | ||
In case `series` is multivariate and `model` is a subclass of `UnivariateForecastingModel`, | ||
an integer index of the component of `series` to be predicted by `model`. | ||
use_full_output_length | ||
In case `model` is a subclass of `TorchForecastingModel`, this argument will be passed along | ||
as argument to the predict method of `model`. | ||
num_predictions: | ||
The number of train/prediction cycles performed in one iteration of expanding window mode. | ||
metric: | ||
|
@@ -337,13 +398,16 @@ def backtest_gridsearch(model_class: type, | |
An untrained 'model_class' instance with the best-performing hyperparameters from the given selection. | ||
""" | ||
|
||
train_series._assert_univariate() | ||
if (val_series is not None): | ||
val_series._assert_univariate() | ||
raise_if_not(train_series.width == val_series.width, "Training and validation series require the same" | ||
" number of components.", logger) | ||
|
||
raise_if_not((fcast_horizon_n is None) ^ (val_series is None), | ||
"Please pass exactly one of the arguments 'forecast_horizon_n' or 'val_series'.", logger) | ||
|
||
fit_kwargs, predict_kwargs = _create_parameter_dicts(model_class(), target_indices, component_index, | ||
use_full_output_length) | ||
|
||
if val_series is None: | ||
backtest_start_time = train_series.end_time() - (num_predictions + fcast_horizon_n) * train_series.freq() | ||
min_error = float('inf') | ||
|
@@ -358,11 +422,12 @@ def backtest_gridsearch(model_class: type, | |
param_combination_dict = dict(list(zip(parameters.keys(), param_combination))) | ||
model = model_class(**param_combination_dict) | ||
if val_series is None: # expanding window mode | ||
backtest_forecast = backtest_forecasting(train_series, model, backtest_start_time, fcast_horizon_n) | ||
backtest_forecast = backtest_forecasting(train_series, model, backtest_start_time, fcast_horizon_n, | ||
target_indices, component_index, use_full_output_length) | ||
error = metric(backtest_forecast, train_series) | ||
else: # split mode | ||
model.fit(train_series) | ||
error = metric(model.predict(len(val_series)), val_series) | ||
model.fit(train_series, **fit_kwargs) | ||
error = metric(model.predict(len(val_series)), val_series, **predict_kwargs) | ||
if error < min_error: | ||
min_error = error | ||
best_param_combination = param_combination_dict | ||
|
@@ -376,8 +441,9 @@ def explore_models(train_series: TimeSeries, | |
metric: Callable[[TimeSeries, TimeSeries], float] = metrics.mape, | ||
model_parameter_tuples: Optional[list] = None, | ||
plot_width: int = 3, | ||
verbose: bool = True): | ||
""" A function for exploring the suitability of multiple models on a given train/validation/test split. | ||
verbose: bool = False): | ||
""" A function for exploring the suitability of multiple models on a given train/validation/test split | ||
of a univariate series. | ||
|
||
This funtion iterates through a list of models, training each on `train_series` and `val_series` | ||
and evaluating them on `test_series`. Models with free hyperparameters are first | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I believe this is what @guillaumeraille was asking on the daily in the morning.
@pennfranc this looks good, think we can also somehow make it automatic?