-
Notifications
You must be signed in to change notification settings - Fork 1.7k
/
test_classification_tuning.py
125 lines (111 loc) · 3.41 KB
/
test_classification_tuning.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
import os
import pandas as pd
import pytest
import pycaret.classification
import pycaret.datasets
from pycaret.utils.generic import can_early_stop
os.environ["TUNE_DISABLE_AUTO_CALLBACK_LOGGERS"] = "1"
os.environ["TUNE_MAX_LEN_IDENTIFIER"] = "1"
if "CI" in os.environ:
pytest.skip("Skipping test module on CI", allow_module_level=True)
@pytest.mark.skip(reason="no way of currently testing this")
def test_classification_tuning():
# loading dataset
data = pycaret.datasets.get_data("juice")
assert isinstance(data, pd.DataFrame)
# init setup
pycaret.classification.setup(
data,
target="Purchase",
train_size=0.7,
fold=2,
html=False,
session_id=123,
n_jobs=1,
)
models = pycaret.classification.compare_models(
turbo=False, n_select=100, verbose=False
)
models.append(pycaret.classification.stack_models(models[:3], verbose=False))
models.append(pycaret.classification.ensemble_model(models[0], verbose=False))
for model in models:
print(f"Testing model {model}")
if "Dummy" in str(model):
continue
pycaret.classification.tune_model(
model,
fold=2,
n_iter=2,
search_library="scikit-learn",
search_algorithm="random",
early_stopping=False,
)
pycaret.classification.tune_model(
model,
fold=2,
n_iter=2,
search_library="scikit-optimize",
search_algorithm="bayesian",
early_stopping=False,
)
pycaret.classification.tune_model(
model,
fold=2,
n_iter=2,
search_library="optuna",
search_algorithm="tpe",
early_stopping=False,
)
# TODO: Enable ray after fix is released
# pycaret.classification.tune_model(
# model,
# fold=2,
# n_iter=2,
# search_library="tune-sklearn",
# search_algorithm="random",
# early_stopping=False,
# )
# pycaret.classification.tune_model(
# model,
# fold=2,
# n_iter=2,
# search_library="tune-sklearn",
# search_algorithm="optuna",
# early_stopping=False,
# )
pycaret.classification.tune_model(
model,
fold=2,
n_iter=2,
search_library="optuna",
search_algorithm="tpe",
early_stopping="asha",
)
# pycaret.classification.tune_model(
# model,
# fold=2,
# n_iter=2,
# search_library="tune-sklearn",
# search_algorithm="hyperopt",
# early_stopping="asha",
# )
# pycaret.classification.tune_model(
# model,
# fold=2,
# n_iter=2,
# search_library="tune-sklearn",
# search_algorithm="bayesian",
# early_stopping="asha",
# )
if can_early_stop(model, True, True, True, {}):
pycaret.classification.tune_model(
model,
fold=2,
n_iter=2,
search_library="tune-sklearn",
search_algorithm="bohb",
early_stopping=True,
)
assert 1 == 1
if __name__ == "__main__":
test_classification_tuning()