358 Reduce execution time for unit tests (#1098)

.codecov.yml

@@ -9,4 +9,6 @@ ignore:
   - "fedot/sensitivity" # unit test are moved to integration test due to the time restrictions
   - "fedot/utilities" # unit test are moved to integration test due to the time restrictions
   - "fedot/visualisation" # complicated to test in unit-like way, tested in integration tests
-  - "fedot/core/operations/evaluation/automl.py" # require to heavy external dependencies - tested in integration test
+  - "fedot/core/operations/evaluation/automl.py" # require to heavy external dependencies - tested in integration test
+  - "fedot/api/help.py" # unit test are moved to integration test due to the time restrictions
+  - "fedot/core/pipelines/tuning/search_space.py" # unit test are moved to integration test due to the time restrictions

.github/workflows/integration-build.yml

@@ -0,0 +1,32 @@
+name: Integration build
+
+on:
+  schedule:
+    - cron: '0 12 * * *'
+
+jobs:
+  scheduled:
+    runs-on: ubuntu-latest
+    timeout-minutes: 95
+    strategy:
+      matrix:
+        python-version: [ 3.8, 3.9, '3.10' ]
+
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest
+          pip install .[extra]
+          pip install .[examples]
+          pip install .[profilers]
+          pip install pytest-cov
+      - name: Test with pytest
+        run: |
+          pytest --cov=fedot -s test/integration

.github/workflows/manual-build.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   integration_test:
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 95
     strategy:
       matrix:
         python-version: [ 3.8, 3.9, '3.10' ]

.github/workflows/unit-build.yml

@@ -28,8 +28,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install pytest
-        pip install .[extra]
-        pip install .[examples]
+        pip install .
         pip install pytest-cov
     - name: Test with pytest
       run: |

examples/advanced/multitask_classification_regression.py

@@ -1,5 +1,5 @@
-import os
 from datetime import timedelta
+from typing import Tuple
 
 import numpy as np
 import pandas as pd
@@ -14,7 +14,7 @@
 from fedot.core.repository.dataset_types import DataTypesEnum
 from fedot.core.repository.quality_metrics_repository import RegressionMetricsEnum
 from fedot.core.repository.tasks import TaskTypesEnum, Task
-from test.unit.api.test_api_cli_params import project_root_path
+from fedot.core.utils import fedot_project_root
 
 
 def get_multitask_pipeline():
@@ -24,11 +24,11 @@ def get_multitask_pipeline():
     return Pipeline(final_node)
 
 
-def prepare_multitask_data() -> (MultiModalData, MultiModalData):
+def prepare_multitask_data() -> Tuple[MultiModalData, MultiModalData]:
     """ Load data for multitask regression / classification pipeline """
-    ex_data = os.path.join(project_root_path, 'examples/data')
-    train_df = pd.read_csv(os.path.join(ex_data, 'train_synthetic_regression_classification.csv'))
-    test_df = pd.read_csv(os.path.join(ex_data, 'test_synthetic_regression_classification.csv'))
+    ex_data = fedot_project_root().joinpath('examples/data')
+    train_df = pd.read_csv(ex_data.joinpath('train_synthetic_regression_classification.csv'))
+    test_df = pd.read_csv(ex_data.joinpath('test_synthetic_regression_classification.csv'))
 
     # Data for classification
     class_task = Task(TaskTypesEnum.classification)
@@ -66,12 +66,14 @@ def launch_multitask_example(with_tuning: bool = False):
     multitask_pipeline = get_multitask_pipeline()
 
     if with_tuning:
-        tuner = TunerBuilder(train_input.task)\
-            .with_tuner(SimultaneousTuner)\
-            .with_metric(RegressionMetricsEnum.MAE)\
-            .with_iterations(100)\
-            .with_timeout(timedelta(minutes=2))\
+        tuner = (
+            TunerBuilder(train_input.task)
+            .with_tuner(SimultaneousTuner)
+            .with_metric(RegressionMetricsEnum.MAE)
+            .with_iterations(100)
+            .with_timeout(timedelta(minutes=2))
             .build(train_input)
+        )
         multitask_pipeline = tuner.tune(multitask_pipeline)
 
     multitask_pipeline.fit(train_input)

examples/simple/multitask_classification_regression_api.py

@@ -1,15 +1,16 @@
 import os
+from typing import Tuple
 
 import numpy as np
 import pandas as pd
 
 from fedot.api.main import Fedot
-from test.unit.api.test_api_cli_params import project_root_path
+from fedot.core.utils import fedot_project_root
 
 
-def load_train_test_dataframes() -> (pd.DataFrame, pd.DataFrame):
+def load_train_test_dataframes() -> Tuple[dict, dict, dict]:
     """ Load data for multitask regression / classification problem """
-    data_path = os.path.join(project_root_path, 'examples/data')
+    data_path = fedot_project_root().joinpath('examples/data')
     train_df = pd.read_csv(os.path.join(data_path, 'train_synthetic_regression_classification.csv'))
     test_df = pd.read_csv(os.path.join(data_path, 'test_synthetic_regression_classification.csv'))
 

fedot/api/help.py

@@ -15,14 +15,18 @@ def print_models_info(task_name):
 
     # Filter operations
     repository_operations_list = _filter_operations_by_type(repository, task)
+    search_space = PipelineSearchSpace()
     for model in repository_operations_list:
         if model.id != 'custom':
-            hyperparameters = PipelineSearchSpace().get_operation_parameter_range(str(model.id))
+            hyperparameters = search_space.get_operation_parameter_range(str(model.id))
             implementation_info = model.current_strategy(task)(model.id).implementation_info
-            print(f"Model name - '{model.id}'")
-            print(f"Available hyperparameters to optimize with tuner - {hyperparameters}")
-            print(f"Strategy implementation - {model.current_strategy(task)}")
-            print(f"Model implementation - {implementation_info}\n")
+            info_lst = [
+                f"Model name - '{model.id}'",
+                f"Available hyperparameters to optimize with tuner - {hyperparameters}",
+                f"Strategy implementation - {model.current_strategy(task)}",
+                f"Model implementation - {implementation_info}\n"
+            ]
+            print('\n'.join(info_lst))
 
 
 def print_data_operations_info(task_name):
@@ -34,15 +38,18 @@ def print_data_operations_info(task_name):
     task = _get_task_by_name(task_name)
 
     repository = OperationTypesRepository(operation_type='data_operation')
-    # Filter operations
     repository_operations_list = _filter_operations_by_type(repository, task)
+    search_space = PipelineSearchSpace()
     for operation in repository_operations_list:
-        hyperparameters = PipelineSearchSpace().get_operation_parameter_range(str(operation.id))
+        hyperparameters = search_space.get_operation_parameter_range(str(operation.id))
         implementation_info = operation.current_strategy(task)(operation.id).implementation_info
-        print(f"Data operation name - '{operation.id}'")
-        print(f"Available hyperparameters to optimize with tuner - {hyperparameters}")
-        print(f"Strategy implementation - {operation.current_strategy(task)}")
-        print(f"Operation implementation - {implementation_info}\n")
+        info_lst = [
+            f"Data operation name - '{operation.id}'",
+            f"Available hyperparameters to optimize with tuner - {hyperparameters}",
+            f"Strategy implementation - {operation.current_strategy(task)}",
+            f"Operation implementation - {implementation_info}\n"
+        ]
+        print('\n'.join(info_lst))
 
 
 def _filter_operations_by_type(repository, task):

fedot/core/pipelines/tuning/search_space.py

@@ -1,3 +1,5 @@
+from typing import Optional, Dict, Tuple, Callable, List
+
 import numpy as np
 from golem.core.tuning.search_space import SearchSpace
 from hyperopt import hp
@@ -13,7 +15,7 @@ class PipelineSearchSpace(SearchSpace):
     """
 
     def __init__(self,
-                 custom_search_space: dict = None,
+                 custom_search_space: Optional[Dict[str, Dict[str, Tuple[Callable, List]]]] = None,
                  replace_default_search_space: bool = False):
         self.custom_search_space = custom_search_space
         self.replace_default_search_space = replace_default_search_space
@@ -291,11 +293,10 @@ def get_parameters_dict(self):
         }
 
         if self.custom_search_space is not None:
-            for operation in self.custom_search_space.keys():
-                if self.replace_default_search_space:
-                    parameters_per_operation[operation] = self.custom_search_space[operation]
-                else:
-                    for key, value in self.custom_search_space[operation].items():
-                        parameters_per_operation[operation][key] = value
+            if self.replace_default_search_space:
+                parameters_per_operation.update(self.custom_search_space)
+            else:
+                for operation_name, operation_dct in self.custom_search_space.items():
+                    parameters_per_operation[operation_name].update(operation_dct)
 
         return parameters_per_operation

test/integration/api/test_api_cli_params.py

@@ -0,0 +1,39 @@
+from typing import List
+
+from fedot.api.fedot_cli import create_parser, separate_argparse_to_fedot, preprocess_keys, run_fedot, \
+    arguments_dicts
+from fedot.core.utils import fedot_project_root
+
+
+def call_cli_with_parameters(call_string: List[str]):
+    """ Function that imitates argparse api call"""
+    parser = create_parser(arguments_dicts)
+    parameters = parser.parse_args(call_string)
+    main_params, fit_params = separate_argparse_to_fedot(parameters)
+    preprocess_keys(main_params)
+    preprocess_keys(fit_params)
+    predictions = run_fedot(parameters, main_params, fit_params, save_predictions=False)
+    return predictions
+
+
+def test_cli_with_parameters():
+    """ Test all parameters used in cli are available from api"""
+    project_root_path = fedot_project_root()
+    ts_train_path = project_root_path.joinpath('test/data/simple_time_series.csv')
+    ts_call = (
+        f'--problem ts_forecasting --preset fast_train --timeout 0.1 --depth 3 --arity 3 '
+        '--popsize 3 --gen_num 5 --opers lagged linear ridge --tuning 0 '
+        f'--cv_folds 2 --val_bl 2 --target sea_height --train {ts_train_path} '
+        f'--test {ts_train_path} --for_len 10'
+    ).split()
+    class_train_path = project_root_path.joinpath('test/data/simple_classification.csv')
+    class_call = (
+        f'--problem classification --train {class_train_path} --test {class_train_path} --target Y '
+        '--preset fast_train --timeout 0.1 --depth 3 --arity 3 '
+        '--popsize 3 --gen_num 5 --tuning 1'
+    ).split()
+
+    ts_predictions = call_cli_with_parameters(ts_call)
+    assert ts_predictions is not None
+    class_predictions = call_cli_with_parameters(class_call)
+    assert class_predictions is not None

test/unit/api/test_api_utils.py → test/integration/api/test_api_utils.py

@@ -13,7 +13,7 @@
 from fedot.core.pipelines.pipeline import Pipeline
 from fedot.core.repository.tasks import Task, TaskTypesEnum, TsForecastingParams
 from fedot.preprocessing.preprocessing import DataPreprocessor
-from test.unit.api.test_main_api import get_dataset, get_cholesterol_dataset
+from test.integration.api.test_main_api import get_dataset, get_cholesterol_dataset
 from test.unit.tasks.test_classification import get_binary_classification_data