diff --git a/docs_sources/related.md b/docs_sources/related.md
index 9b3efad3..fd11d4c1 100644
--- a/docs_sources/related.md
+++ b/docs_sources/related.md
@@ -52,7 +52,13 @@ Other Automated Machine Learning (AutoML) tools and related projects:
| Xcessiv |
Python |
-Apache-2.0 |
+Apache 2.0 |
A web-based application for quick, scalable, and automated hyper-parameter tuning and stacked ensembling in Python. |
+
+| GAMA |
+Python |
+Apache 2.0 |
+Machine-learning pipeline optimization through asynchronous evaluation based genetic programming. |
+
diff --git a/docs_sources/using.md b/docs_sources/using.md
index 4ff0bd5d..4662456b 100644
--- a/docs_sources/using.md
+++ b/docs_sources/using.md
@@ -298,7 +298,7 @@ See the built-in configurations
CHECKPOINT_FOLDER |
Folder path |
-If supplied, a folder you created, in which tpot will periodically save the best pipeline so far while optimizing.
+If supplied, a folder you created, in which tpot will periodically save pipelines in pareto front so far while optimizing.
This is useful in multiple cases:
diff --git a/requirements.txt b/requirements.txt
index bef337e8..a625890e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ nose==1.3.7
numpy==1.12.1
scikit-learn==0.18.1
scipy==0.19.0
-tqdm==4.11.2
+tqdm==4.26.0
update-checker==0.16
stopit==1.1.1
pandas==0.20.2
diff --git a/setup.py b/setup.py
index 681f45b0..51475064 100644
--- a/setup.py
+++ b/setup.py
@@ -40,7 +40,7 @@ def calculate_version():
'scikit-learn>=0.18.1',
'deap>=1.0',
'update_checker>=0.16',
- 'tqdm>=4.11.2',
+ 'tqdm>=4.26.0',
'stopit>=1.1.1',
'pandas>=0.20.2'],
extras_require={
diff --git a/tests/stacking_estimator_tests.py b/tests/stacking_estimator_tests.py
index 630aea6a..e7b62a60 100644
--- a/tests/stacking_estimator_tests.py
+++ b/tests/stacking_estimator_tests.py
@@ -30,6 +30,8 @@
from sklearn.pipeline import make_pipeline
from tpot_tests import training_features, training_target, training_features_r, training_target_r
from sklearn.model_selection import cross_val_score
+import warnings
+warnings.filterwarnings("ignore")
def test_StackingEstimator_1():
"""Assert that the StackingEstimator returns transformed X with synthetic features in classification."""
@@ -76,8 +78,7 @@ def test_StackingEstimator_3():
# test cv score
cv_score = np.mean(cross_val_score(sklearn_pipeline, training_features, training_target, cv=3, scoring='accuracy'))
-
- known_cv_score = 0.947282375315
+ known_cv_score = 0.9472823753147593
assert np.allclose(known_cv_score, cv_score)
@@ -100,6 +101,6 @@ def test_StackingEstimator_4():
# test cv score
cv_score = np.mean(cross_val_score(sklearn_pipeline, training_features_r, training_target_r, cv=3, scoring='r2'))
- known_cv_score = 0.795877470354
+ known_cv_score = 0.7989564328211737
assert np.allclose(known_cv_score, cv_score)
diff --git a/tests/test_dask_based.py b/tests/test_dask_based.py
index b042edab..b4daf0f4 100644
--- a/tests/test_dask_based.py
+++ b/tests/test_dask_based.py
@@ -3,7 +3,6 @@
https://github.com/DEAP/deap/issues/75
"""
import unittest
-
import nose
from sklearn.datasets import make_classification
@@ -21,25 +20,25 @@ class TestDaskMatches(unittest.TestCase):
def test_dask_matches(self):
with dask.config.set(scheduler='single-threaded'):
for n_jobs in [-1]:
- X, y = make_classification(random_state=0)
+ X, y = make_classification(random_state=42)
a = TPOTClassifier(
- generations=2,
+ generations=0,
population_size=5,
cv=3,
- random_state=0,
+ random_state=42,
n_jobs=n_jobs,
use_dask=False,
)
b = TPOTClassifier(
- generations=2,
+ generations=0,
population_size=5,
cv=3,
- random_state=0,
+ random_state=42,
n_jobs=n_jobs,
use_dask=True,
)
- b.fit(X, y)
a.fit(X, y)
+ b.fit(X, y)
self.assertEqual(a.score(X, y), b.score(X, y))
self.assertEqual(a.pareto_front_fitted_pipelines_.keys(),
diff --git a/tests/tpot_tests.py b/tests/tpot_tests.py
index 63829719..2a50b200 100644
--- a/tests/tpot_tests.py
+++ b/tests/tpot_tests.py
@@ -24,12 +24,12 @@
"""
from tpot import TPOTClassifier, TPOTRegressor
-from tpot.base import TPOTBase, is_notebook
+from tpot.base import TPOTBase
from tpot.driver import float_range
from tpot.gp_types import Output_Array
from tpot.gp_deap import mutNodeReplacement, _wrapped_cross_val_score, pick_two_individuals_eligible_for_crossover, cxOnePoint, varOr, initialize_stats_dict
from tpot.metrics import balanced_accuracy, SCORERS
-from tpot.operator_utils import TPOTOperatorClassFactory, set_sample_weight
+from tpot.operator_utils import TPOTOperatorClassFactory, set_sample_weight, source_decode
from tpot.decorators import pretest_X, pretest_y
from tpot.config.classifier import classifier_config_dict
@@ -63,7 +63,9 @@
from nose.tools import assert_raises, assert_not_equal, assert_greater_equal, assert_equal, assert_in
from driver_tests import captured_output
-from tqdm import tqdm
+with warnings.catch_warnings():
+ warnings.simplefilter('ignore')
+ from tqdm.autonotebook import tqdm
try:
from StringIO import StringIO
@@ -154,13 +156,7 @@ def test_init_custom_parameters():
assert tpot_obj._optimized_pipeline == None
assert tpot_obj._optimized_pipeline_score == None
assert tpot_obj.fitted_pipeline_ == None
- assert tpot_obj._exported_pipeline_text == ""
-
-
-def test_is_notebook():
- """Assert that isnotebook function works as expected."""
- ret = is_notebook()
- assert not ret
+ assert tpot_obj._exported_pipeline_text == []
def test_init_default_scoring():
@@ -527,7 +523,7 @@ def test_score_3():
"""Assert that the TPOTRegressor score function outputs a known score for a fixed pipeline."""
tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error', random_state=72)
tpot_obj._fit_init()
- known_score = -12.1791953611
+ known_score = -11.682841148312662
# Reify pipeline with known score
pipeline_string = (
@@ -590,8 +586,9 @@ def test_sample_weight_func():
np.random.seed(42)
tpot_obj.fitted_pipeline_.fit(training_features_r, training_target_r, **training_target_r_weight_dict)
# Get score from TPOT
- known_score = -11.5790430757
+ known_score = -11.586816877933911
score = tpot_obj.score(testing_features_r, testing_target_r)
+ print(score)
assert np.allclose(cv_score1, cv_score2)
assert not np.allclose(cv_score1, cv_score_weight)
@@ -1005,48 +1002,16 @@ def test_check_periodic_pipeline():
tpot_obj._last_pipeline_write = datetime.now()
sleep(0.11)
tpot_obj._output_best_pipeline_period_seconds = 0.1
- tpot_obj.periodic_checkpoint_folder = './'
- tpot_obj._check_periodic_pipeline()
+ tmpdir = mkdtemp() + '/'
+ tpot_obj.periodic_checkpoint_folder = tmpdir
+ tpot_obj._check_periodic_pipeline(1)
our_file.seek(0)
-
- assert_in('Saving best periodic pipeline to ./pipeline', our_file.read())
+ assert_in('Saving periodic pipeline from pareto front', our_file.read())
# clean up
- for f in os.listdir('./'):
- if search('pipeline_', f):
- os.remove(os.path.join('./', f))
+ rmtree(tmpdir)
def test_check_periodic_pipeline_2():
- """Assert that the _check_periodic_pipeline does not export periodic pipeline if the pipeline has been saved before."""
- tpot_obj = TPOTClassifier(
- random_state=42,
- population_size=1,
- offspring_size=2,
- generations=1,
- verbosity=0,
- config_dict='TPOT light'
- )
- tpot_obj.fit(training_features, training_target)
- with closing(StringIO()) as our_file:
- tpot_obj._file = our_file
- tpot_obj.verbosity = 3
- tpot_obj._last_pipeline_write = datetime.now()
- sleep(0.11)
- tpot_obj._output_best_pipeline_period_seconds = 0.1
- tpot_obj.periodic_checkpoint_folder = './'
- # export once before
- tpot_obj.export('./pipeline_test.py')
- tpot_obj._check_periodic_pipeline()
- our_file.seek(0)
-
- assert_in('Periodic pipeline was not saved, probably saved before...', our_file.read())
- # clean up
- for f in os.listdir('./'):
- if search('pipeline_', f):
- os.remove(os.path.join('./', f))
-
-
-def test_check_periodic_pipeline_3():
"""Assert that the _check_periodic_pipeline rasie StopIteration if self._last_optimized_pareto_front_n_gens >= self.early_stop."""
tpot_obj = TPOTClassifier(
random_state=42,
@@ -1059,9 +1024,9 @@ def test_check_periodic_pipeline_3():
tpot_obj.fit(training_features, training_target)
tpot_obj.early_stop = 3
# will pass
- tpot_obj._check_periodic_pipeline()
+ tpot_obj._check_periodic_pipeline(1)
tpot_obj._last_optimized_pareto_front_n_gens = 3
- assert_raises(StopIteration, tpot_obj._check_periodic_pipeline)
+ assert_raises(StopIteration, tpot_obj._check_periodic_pipeline, 1)
def test_save_periodic_pipeline():
@@ -1081,18 +1046,18 @@ def test_save_periodic_pipeline():
tpot_obj._last_pipeline_write = datetime.now()
sleep(0.11)
tpot_obj._output_best_pipeline_period_seconds = 0.1
- tpot_obj.periodic_checkpoint_folder = './'
- # reset _optimized_pipeline to rasie exception
- tpot_obj._optimized_pipeline = None
+ tmpdir = mkdtemp() + '/'
+ tpot_obj.periodic_checkpoint_folder = tmpdir
+ # reset _pareto_front to rasie exception
+ tpot_obj._pareto_front = None
- tpot_obj._save_periodic_pipeline()
+ tpot_obj._save_periodic_pipeline(1)
our_file.seek(0)
assert_in('Failed saving periodic pipeline, exception', our_file.read())
- # clean up
- for f in os.listdir('./'):
- if search('pipeline_', f):
- os.remove(os.path.join('./', f))
+ #clean up
+ rmtree(tmpdir)
+
def test_save_periodic_pipeline_2():
"""Assert that _save_periodic_pipeline creates the checkpoint folder and exports to it if it didn't exist"""
@@ -1111,20 +1076,49 @@ def test_save_periodic_pipeline_2():
tpot_obj._last_pipeline_write = datetime.now()
sleep(0.11)
tpot_obj._output_best_pipeline_period_seconds = 0.1
- tpot_obj.periodic_checkpoint_folder = './tmp'
- tpot_obj._save_periodic_pipeline()
+ tmpdir = mkdtemp() + '_test/'
+ tpot_obj.periodic_checkpoint_folder = tmpdir
+ tpot_obj._save_periodic_pipeline(1)
our_file.seek(0)
msg = our_file.read()
- expected_filepath_prefix = os.path.join('./tmp', 'pipeline_')
- assert_in('Saving best periodic pipeline to ' + expected_filepath_prefix, msg)
- assert_in('Created new folder to save periodic pipeline: ./tmp', msg)
- # clean up
- for f in os.listdir('./tmp'):
- if search('pipeline_', f):
- os.remove(os.path.join('./tmp', f))
- os.rmdir('./tmp')
+ assert_in('Saving periodic pipeline from pareto front to {}'.format(tmpdir), msg)
+ assert_in('Created new folder to save periodic pipeline: {}'.format(tmpdir), msg)
+
+ #clean up
+ rmtree(tmpdir)
+
+
+def test_check_periodic_pipeline_3():
+ """Assert that the _save_periodic_pipeline does not export periodic pipeline if the pipeline has been saved before."""
+ tpot_obj = TPOTClassifier(
+ random_state=42,
+ population_size=1,
+ offspring_size=2,
+ generations=1,
+ verbosity=0,
+ config_dict='TPOT light'
+ )
+ tpot_obj.fit(training_features, training_target)
+ with closing(StringIO()) as our_file:
+ tpot_obj._file = our_file
+ tpot_obj.verbosity = 3
+ tpot_obj._exported_pipeline_text = []
+ tpot_obj._last_pipeline_write = datetime.now()
+ sleep(0.11)
+ tpot_obj._output_best_pipeline_period_seconds = 0
+ tmpdir = mkdtemp() + '/'
+ tpot_obj.periodic_checkpoint_folder = tmpdir
+ # export once before
+ tpot_obj._save_periodic_pipeline(1)
+
+ tpot_obj._save_periodic_pipeline(2)
+
+ our_file.seek(0)
+ assert_in('Periodic pipeline was not saved, probably saved before...', our_file.read())
+ #clean up
+ rmtree(tmpdir)
def test_fit_predict():
@@ -1768,6 +1762,29 @@ def test_sparse_matrix_5():
tpot_obj.fit(sparse_features, sparse_target)
+def test_source_decode():
+ """Assert that the source_decode can decode operator source and import operator class."""
+ import_str, op_str, op_obj = source_decode("sklearn.linear_model.LogisticRegression")
+ from sklearn.linear_model import LogisticRegression
+ assert import_str == "sklearn.linear_model"
+ assert op_str == "LogisticRegression"
+ assert op_obj == LogisticRegression
+
+
+def test_source_decode_2():
+ """Assert that the source_decode return None when sourcecode is not available."""
+ import_str, op_str, op_obj = source_decode("sklearn.linear_model.LogisticReg")
+ from sklearn.linear_model import LogisticRegression
+ assert import_str == "sklearn.linear_model"
+ assert op_str == "LogisticReg"
+ assert op_obj is None
+
+
+def test_source_decode_3():
+ """Assert that the source_decode raise ImportError when sourcecode is not available and verbose=3."""
+ assert_raises(ImportError, source_decode, "sklearn.linear_model.LogisticReg", 3)
+
+
def test_tpot_operator_factory_class():
"""Assert that the TPOT operators class factory."""
test_config_dict = {
diff --git a/tpot/_version.py b/tpot/_version.py
index 7487539f..ab687bdd 100644
--- a/tpot/_version.py
+++ b/tpot/_version.py
@@ -23,4 +23,4 @@
"""
-__version__ = '0.9.5'
+__version__ = '0.9.6'
diff --git a/tpot/base.py b/tpot/base.py
index 0e96b570..959569bf 100644
--- a/tpot/base.py
+++ b/tpot/base.py
@@ -71,6 +71,9 @@
from .metrics import SCORERS
from .gp_types import Output_Array
from .gp_deap import eaMuPlusLambda, mutNodeReplacement, _wrapped_cross_val_score, cxOnePoint
+with warnings.catch_warnings():
+ warnings.simplefilter('ignore')
+ from tqdm.autonotebook import tqdm
# hot patch for Windows: solve the problem of crashing python after Ctrl + C in Windows OS
# https://github.com/ContinuumIO/anaconda-issues/issues/905
@@ -93,25 +96,6 @@ def handler(dwCtrlType, hook_sigint=_thread.interrupt_main):
win32api.SetConsoleCtrlHandler(handler, 1)
-def is_notebook():
- """Check if TPOT is running in Jupyter notebook.
- Returns
- -------
- True: TPOT is running in Jupyter notebook
- False: TPOT is running in other terminals
- """
- try:
- from IPython import get_ipython
- shell = get_ipython().__class__.__name__
- # if shell == 'TerminalInteractiveShell', then Terminal running IPython
- return shell == 'ZMQInteractiveShell'
- except:
- return False
-
-if is_notebook():
- from tqdm import tqdm_notebook as tqdm
-else:
- from tqdm import tqdm
class TPOTBase(BaseEstimator):
@@ -245,7 +229,7 @@ def __init__(self, generations=100, population_size=100, offspring_size=None,
See `avoid repeated work `__
for more details.
periodic_checkpoint_folder: path string, optional (default: None)
- If supplied, a folder in which tpot will periodically save the best pipeline so far while optimizing.
+ If supplied, a folder in which tpot will periodically save pipelines in pareto front so far while optimizing.
Currently once per generation but not more often than once per 30 seconds.
Useful in multiple cases:
Sudden death before tpot could save optimized pipeline
@@ -460,7 +444,7 @@ def _fit_init(self):
self._optimized_pipeline = None
self._optimized_pipeline_score = None
- self._exported_pipeline_text = ""
+ self._exported_pipeline_text = []
self.fitted_pipeline_ = None
self._fitted_imputer = None
self._imputed = False
@@ -482,7 +466,8 @@ def _fit_init(self):
key,
self._config_dict[key],
BaseClass=Operator,
- ArgBaseClass=ARGType
+ ArgBaseClass=ARGType,
+ verbose=self.verbosity
)
if op_class:
self.operators.append(op_class)
@@ -930,32 +915,55 @@ def clean_pipeline_string(self, individual):
return pretty
- def _check_periodic_pipeline(self):
- """If enough time has passed, save a new optimized pipeline.
+ def _check_periodic_pipeline(self, gen):
+ """If enough time has passed, save a new optimized pipeline. Currently used in the per generation hook in the optimization loop.
+ Parameters
+ ----------
+ gen: int
+ Generation number
- Currently used in the per generation hook in the optimization loop.
+ Returns
+ -------
+ None
"""
self._update_top_pipeline()
if self.periodic_checkpoint_folder is not None:
total_since_last_pipeline_save = (datetime.now() - self._last_pipeline_write).total_seconds()
if total_since_last_pipeline_save > self._output_best_pipeline_period_seconds:
self._last_pipeline_write = datetime.now()
- self._save_periodic_pipeline()
+ self._save_periodic_pipeline(gen)
if self.early_stop is not None:
if self._last_optimized_pareto_front_n_gens >= self.early_stop:
raise StopIteration("The optimized pipeline was not improved after evaluating {} more generations. "
"Will end the optimization process.\n".format(self.early_stop))
- def _save_periodic_pipeline(self):
+ def _save_periodic_pipeline(self, gen):
try:
self._create_periodic_checkpoint_folder()
- filename = os.path.join(self.periodic_checkpoint_folder, 'pipeline_{}.py'.format(datetime.now().strftime('%Y.%m.%d_%H-%M-%S')))
- did_export = self.export(filename, skip_if_repeated=True)
- if not did_export:
- self._update_pbar(pbar_num=0, pbar_msg='Periodic pipeline was not saved, probably saved before...')
- else:
- self._update_pbar(pbar_num=0, pbar_msg='Saving best periodic pipeline to {}'.format(filename))
+ for pipeline, pipeline_scores in zip(self._pareto_front.items, reversed(self._pareto_front.keys)):
+ idx = self._pareto_front.items.index(pipeline)
+ pareto_front_pipeline_score = pipeline_scores.wvalues[1]
+ sklearn_pipeline_str = generate_pipeline_code(expr_to_tree(pipeline, self._pset), self.operators)
+ to_write = export_pipeline(pipeline,
+ self.operators, self._pset,
+ self._imputed, pareto_front_pipeline_score,
+ self.random_state)
+ # dont export a pipeline you had
+ if self._exported_pipeline_text.count(sklearn_pipeline_str):
+ self._update_pbar(pbar_num=0, pbar_msg='Periodic pipeline was not saved, probably saved before...')
+ else:
+ filename = os.path.join(self.periodic_checkpoint_folder,
+ 'pipeline_gen_{}_idx_{}_{}.py'.format(gen,
+ idx ,
+ datetime.now().strftime('%Y.%m.%d_%H-%M-%S')
+ )
+ )
+ self._update_pbar(pbar_num=0, pbar_msg='Saving periodic pipeline from pareto front to {}'.format(filename))
+ with open(filename, 'w') as output_file:
+ output_file.write(to_write)
+ self._exported_pipeline_text.append(sklearn_pipeline_str)
+
except Exception as e:
self._update_pbar(pbar_num=0, pbar_msg='Failed saving periodic pipeline, exception:\n{}'.format(str(e)[:250]))
@@ -969,16 +977,14 @@ def _create_periodic_checkpoint_folder(self):
else:
raise ValueError('Failed creating the periodic_checkpoint_folder:\n{}'.format(e))
- def export(self, output_file_name, skip_if_repeated=False):
+
+ def export(self, output_file_name):
"""Export the optimized pipeline as Python code.
Parameters
----------
output_file_name: string
String containing the path and file name of the desired output file
- skip_if_repeated: boolean
- If True, skip the actual writing if a pipeline
- code would be identical to the last pipeline exported
Returns
-------
@@ -989,17 +995,14 @@ def export(self, output_file_name, skip_if_repeated=False):
if self._optimized_pipeline is None:
raise RuntimeError('A pipeline has not yet been optimized. Please call fit() first.')
- to_write = export_pipeline(self._optimized_pipeline, self.operators, self._pset, self._imputed, self._optimized_pipeline_score, self.random_state)
-
- # dont export a pipeline you just had
- if skip_if_repeated and (self._exported_pipeline_text == to_write):
- return False
+ to_write = export_pipeline(self._optimized_pipeline,
+ self.operators, self._pset,
+ self._imputed, self._optimized_pipeline_score,
+ self.random_state)
with open(output_file_name, 'w') as output_file:
output_file.write(to_write)
- self._exported_pipeline_text = to_write
- return True
def _impute_values(self, features):
"""Impute missing values in a feature set.
@@ -1203,6 +1206,7 @@ def _evaluate_individuals(self, individuals, features, target, sample_weight=Non
)
result_score_list = []
+
# Don't use parallelization if n_jobs==1
if self._n_jobs == 1 and not self.use_dask:
for sklearn_pipeline in sklearn_pipeline_list:
@@ -1211,8 +1215,8 @@ def _evaluate_individuals(self, individuals, features, target, sample_weight=Non
result_score_list = self._update_val(val, result_score_list)
else:
if self.use_dask:
+ self._stop_by_max_time_mins()
import dask
-
result_score_list = [
partial_wrapped_cross_val_score(sklearn_pipeline=sklearn_pipeline)
for sklearn_pipeline in sklearn_pipeline_list
diff --git a/tpot/builtins/feature_transformers.py b/tpot/builtins/feature_transformers.py
index 5cf52b93..b6be141d 100644
--- a/tpot/builtins/feature_transformers.py
+++ b/tpot/builtins/feature_transformers.py
@@ -138,7 +138,7 @@ def fit(self, X, y=None):
def transform(self, X):
- """Select continuous features and transform them using OneHotEncoder.
+ """Select continuous features and transform them using PCA.
Parameters
----------
diff --git a/tpot/config/regressor.py b/tpot/config/regressor.py
index 1d09c035..9278074f 100644
--- a/tpot/config/regressor.py
+++ b/tpot/config/regressor.py
@@ -57,8 +57,7 @@
'sklearn.ensemble.AdaBoostRegressor': {
'n_estimators': [100],
'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.],
- 'loss': ["linear", "square", "exponential"],
- 'max_depth': range(1, 11)
+ 'loss': ["linear", "square", "exponential"]
},
'sklearn.tree.DecisionTreeRegressor': {
diff --git a/tpot/gp_deap.py b/tpot/gp_deap.py
index 6757ef52..d507b4dd 100644
--- a/tpot/gp_deap.py
+++ b/tpot/gp_deap.py
@@ -239,9 +239,7 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar,
# Begin the generational process
for gen in range(1, ngen + 1):
- # after each population save a periodic pipeline
- if per_generation_function is not None:
- per_generation_function()
+
# Vary the population
offspring = varOr(population, toolbox, lambda_, cxpb, mutpb)
@@ -288,6 +286,10 @@ def eaMuPlusLambda(population, toolbox, mu, lambda_, cxpb, mutpb, ngen, pbar,
)
)
pbar.write('')
+
+ # after each population save a periodic pipeline
+ if per_generation_function is not None:
+ per_generation_function(gen)
# Update the statistics with the new population
record = stats.compile(population) if stats is not None else {}
diff --git a/tpot/operator_utils.py b/tpot/operator_utils.py
index 47c49888..b92b6494 100644
--- a/tpot/operator_utils.py
+++ b/tpot/operator_utils.py
@@ -43,13 +43,17 @@ class ARGType(object):
pass
-def source_decode(sourcecode):
+def source_decode(sourcecode, verbose=0):
"""Decode operator source and import operator class.
Parameters
----------
sourcecode: string
a string of operator source (e.g 'sklearn.feature_selection.RFE')
+ verbose: int, optional (default: 0)
+ How much information TPOT communicates while it's running.
+ 0 = none, 1 = minimal, 2 = high, 3 = all.
+ if verbose > 2 then ImportError will rasie during initialization
Returns
@@ -71,8 +75,11 @@ def source_decode(sourcecode):
else:
exec('from {} import {}'.format(import_str, op_str))
op_obj = eval(op_str)
- except ImportError:
- print('Warning: {} is not available and will not be used by TPOT.'.format(sourcecode))
+ except Exception as e:
+ if verbose > 2:
+ raise ImportError('Error: could not import {}.\n{}'.format(sourcecode, e))
+ else:
+ print('Warning: {} is not available and will not be used by TPOT.'.format(sourcecode))
op_obj = None
return import_str, op_str, op_obj
@@ -127,7 +134,7 @@ def ARGTypeClassFactory(classname, prange, BaseClass=ARGType):
return type(classname, (BaseClass,), {'values': prange})
-def TPOTOperatorClassFactory(opsourse, opdict, BaseClass=Operator, ArgBaseClass=ARGType):
+def TPOTOperatorClassFactory(opsourse, opdict, BaseClass=Operator, ArgBaseClass=ARGType, verbose=0):
"""Dynamically create operator class.
Parameters
@@ -144,6 +151,10 @@ def TPOTOperatorClassFactory(opsourse, opdict, BaseClass=Operator, ArgBaseClass=
inherited BaseClass for operator
ArgBaseClass: Class
inherited BaseClass for parameter
+ verbose: int, optional (default: 0)
+ How much information TPOT communicates while it's running.
+ 0 = none, 1 = minimal, 2 = high, 3 = all.
+ if verbose > 2 then ImportError will rasie during initialization
Returns
-------
@@ -156,7 +167,7 @@ def TPOTOperatorClassFactory(opsourse, opdict, BaseClass=Operator, ArgBaseClass=
class_profile = {}
dep_op_list = {} # list of nested estimator/callable function
dep_op_type = {} # type of nested estimator/callable function
- import_str, op_str, op_obj = source_decode(opsourse)
+ import_str, op_str, op_obj = source_decode(opsourse, verbose=verbose)
if not op_obj:
return None, None
@@ -190,7 +201,7 @@ def op_type(cls):
arg_types.append(ARGTypeClassFactory(classname, prange, ArgBaseClass))
else:
for dkey, dval in prange.items():
- dep_import_str, dep_op_str, dep_op_obj = source_decode(dkey)
+ dep_import_str, dep_op_str, dep_op_obj = source_decode(dkey, verbose=verbose)
if dep_import_str in import_hash:
import_hash[import_str].append(dep_op_str)
else:
|