Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

AMLB-specific test case added #1274

Open
wants to merge 18 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions fedot/api/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,12 @@ def get_metrics(self,
data_type=self.train_data.data_type)
else:
self.test_data.target = target[:len(self.prediction.predict)]
elif not len(self.test_data.target):
self.test_data = self.data_processor.define_data(
target=self.train_data.target[:len(self.test_data.features)],
features=self.test_data.features,
is_predict=True,
)

metrics = ensure_wrapped_in_sequence(metric_names) if metric_names else self.metrics
metric_names = [str(metric) for metric in metrics]
Expand Down
3 changes: 2 additions & 1 deletion fedot/preprocessing/preprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,8 @@ def apply_inverse_target_encoding(self, column_to_transform: np.ndarray) -> np.n
# There is no need to perform converting (it was performed already)
return column_to_transform
# It is needed to apply fitted encoder to apply inverse transformation
transformed = self.target_encoders[main_target_source_name].inverse_transform(column_to_transform)
transformed = self.target_encoders[main_target_source_name].inverse_transform(
column_to_transform.astype(int))

# Convert one-dimensional array into column
if len(transformed.shape) == 1:
Expand Down
Binary file added test/data/amlb/target_y.npy
Binary file not shown.
Binary file added test/data/amlb/test_australian_fold7.npy
Binary file not shown.
Binary file added test/data/amlb/testy_australian_fold7.npy
Binary file not shown.
Binary file added test/data/amlb/train_australian_fold7.npy
Binary file not shown.
Binary file added test/data/amlb/trainy_australian_fold7.npy
Binary file not shown.
29 changes: 29 additions & 0 deletions test/integration/api/test_main_api.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import os
import shutil
from copy import deepcopy
from pathlib import Path

import numpy as np
import pandas as pd
Expand All @@ -14,6 +15,7 @@
from fedot.core.pipelines.node import PipelineNode
from fedot.core.pipelines.pipeline import Pipeline
from fedot.core.repository.tasks import TsForecastingParams
from fedot.core.utils import fedot_project_root
from test.data.datasets import get_dataset, get_multimodal_ts_data, load_categorical_unimodal, \
load_categorical_multidata
from test.unit.common_tests import is_predict_ignores_target
Expand Down Expand Up @@ -324,3 +326,30 @@ def test_forecast_with_not_ts_problem():
model.fit(train_data, predefined_model='auto')
with pytest.raises(ValueError):
model.forecast(pre_history=test_data)


@pytest.mark.parametrize(
'initial_assumption, timeout', [
(Pipeline(
PipelineNode(
'catboost', nodes_from=[
PipelineNode(
'resample', nodes_from=[
PipelineNode('scaling')])])), 0.001), (None, 5.0)])
def test_api_for_amlb(initial_assumption, timeout):
amlb_data = Path(fedot_project_root(), 'test', 'data', 'amlb')
x_train = np.load(str(Path(amlb_data, 'train_australian_fold7.npy')))
y_train = np.load(str(Path(amlb_data, 'target_y.npy')), allow_pickle=True) # real target from AMLB
x_test = np.load(str(Path(amlb_data, 'test_australian_fold7.npy')))
# TODO resample add
training_params = {"preset": "best_quality", "n_jobs": -1}

fedot = Fedot(problem='classification', timeout=timeout, metric='roc_auc', seed=0,
max_pipeline_fit_time=1, **training_params, initial_assumption=initial_assumption)

fedot.fit(features=x_train, target=y_train)

predictions = fedot.predict(features=x_test)
probabilities = fedot.predict_proba(features=x_test, probs_for_all_classes=True)
assert predictions is not None
assert probabilities is not None
19 changes: 19 additions & 0 deletions test/unit/api/test_main_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,25 @@ def test_baseline_with_api():
assert baseline_metrics['f1'] > 0


def test_specific_baseline_with_api():
"""
Check that ``get_metrics`` works correctly if ``target`` is defined as a string and ``test_data`` doesn't contain it
"""
df = pd.read_csv(fedot_project_root().joinpath("test/data/simple_classification.csv"))
train_data, test_data = np.array_split(df, 2)
test_data.drop("Y", axis=1, inplace=True)

baseline_model = Fedot(problem="classification", metric=["f1"])

baseline_model.fit(features=train_data, target="Y", predefined_model="auto")

prediction = baseline_model.predict(features=test_data)
assert len(prediction) == len(test_data)

baseline_metrics = baseline_model.get_metrics()
assert baseline_metrics["f1"]


def test_forecast_with_multivariate_ts():
forecast_length = 2

Expand Down
Loading