diff --git a/auto_ml/predictor.py b/auto_ml/predictor.py index 43b7eb9..74c1f66 100644 --- a/auto_ml/predictor.py +++ b/auto_ml/predictor.py @@ -175,8 +175,11 @@ def _construct_pipeline(self, model_name='LogisticRegression', trained_pipeline= if prediction_interval is not False: params = {} - params['objective'] = 'quantile' + params['loss'] = 'quantile' params['alpha'] = prediction_interval + params['n_estimators'] = 100 + params['learning_rate'] = 0.15 + params.update(self.prediction_interval_params) training_prediction_intervals = True elif feature_learning == False: @@ -308,7 +311,7 @@ def _consolidate_pipeline(self, transformation_pipeline, final_model=None): return trained_pipeline_without_feature_selection - def set_params_and_defaults(self, X_df, user_input_func=None, optimize_final_model=None, write_gs_param_results_to_file=True, perform_feature_selection=None, verbose=True, X_test=None, y_test=None, ml_for_analytics=True, take_log_of_y=None, model_names=None, perform_feature_scaling=True, calibrate_final_model=False, _scorer=None, scoring=None, verify_features=False, training_params=None, grid_search_params=None, compare_all_models=False, cv=2, feature_learning=False, fl_data=None, optimize_feature_learning=False, train_uncertainty_model=None, uncertainty_data=None, uncertainty_delta=None, uncertainty_delta_units=None, calibrate_uncertainty=False, uncertainty_calibration_settings=None, uncertainty_calibration_data=None, uncertainty_delta_direction='both', advanced_analytics=True, analytics_config=None, prediction_intervals=None, predict_intervals=None, ensemble_config=None, trained_transformation_pipeline=None, transformed_X=None, transformed_y=None, return_transformation_pipeline=False, X_test_already_transformed=False, skip_feature_responses=None): + def set_params_and_defaults(self, X_df, user_input_func=None, optimize_final_model=None, write_gs_param_results_to_file=True, perform_feature_selection=None, verbose=True, X_test=None, y_test=None, ml_for_analytics=True, take_log_of_y=None, model_names=None, perform_feature_scaling=True, calibrate_final_model=False, _scorer=None, scoring=None, verify_features=False, training_params=None, grid_search_params=None, compare_all_models=False, cv=2, feature_learning=False, fl_data=None, optimize_feature_learning=False, train_uncertainty_model=None, uncertainty_data=None, uncertainty_delta=None, uncertainty_delta_units=None, calibrate_uncertainty=False, uncertainty_calibration_settings=None, uncertainty_calibration_data=None, uncertainty_delta_direction='both', advanced_analytics=True, analytics_config=None, prediction_intervals=None, predict_intervals=None, ensemble_config=None, trained_transformation_pipeline=None, transformed_X=None, transformed_y=None, return_transformation_pipeline=False, X_test_already_transformed=False, skip_feature_responses=None, prediction_interval_params=None): self.user_input_func = user_input_func self.optimize_final_model = optimize_final_model @@ -416,6 +419,10 @@ def set_params_and_defaults(self, X_df, user_input_func=None, optimize_final_mod if predict_intervals is not None and prediction_intervals is None: prediction_intervals = predict_intervals + if prediction_interval_params is None: + self.prediction_interval_params = {} + else: + self.prediction_interval_params = prediction_interval_params if prediction_intervals is None: self.calculate_prediction_intervals = False @@ -595,9 +602,9 @@ def fit_feature_learning_and_transformation_pipeline(self, X_df, fl_data, y): return X_df - def train(self, raw_training_data, user_input_func=None, optimize_final_model=None, write_gs_param_results_to_file=True, perform_feature_selection=None, verbose=True, X_test=None, y_test=None, ml_for_analytics=True, take_log_of_y=None, model_names=None, perform_feature_scaling=None, calibrate_final_model=False, _scorer=None, scoring=None, verify_features=False, training_params=None, grid_search_params=None, compare_all_models=False, cv=2, feature_learning=False, fl_data=None, optimize_feature_learning=False, train_uncertainty_model=False, uncertainty_data=None, uncertainty_delta=None, uncertainty_delta_units=None, calibrate_uncertainty=False, uncertainty_calibration_settings=None, uncertainty_calibration_data=None, uncertainty_delta_direction=None, advanced_analytics=None, analytics_config=None, prediction_intervals=None, predict_intervals=None, ensemble_config=None, trained_transformation_pipeline=None, transformed_X=None, transformed_y=None, return_transformation_pipeline=False, X_test_already_transformed=False, skip_feature_responses=None): + def train(self, raw_training_data, user_input_func=None, optimize_final_model=None, write_gs_param_results_to_file=True, perform_feature_selection=None, verbose=True, X_test=None, y_test=None, ml_for_analytics=True, take_log_of_y=None, model_names=None, perform_feature_scaling=None, calibrate_final_model=False, _scorer=None, scoring=None, verify_features=False, training_params=None, grid_search_params=None, compare_all_models=False, cv=2, feature_learning=False, fl_data=None, optimize_feature_learning=False, train_uncertainty_model=False, uncertainty_data=None, uncertainty_delta=None, uncertainty_delta_units=None, calibrate_uncertainty=False, uncertainty_calibration_settings=None, uncertainty_calibration_data=None, uncertainty_delta_direction=None, advanced_analytics=None, analytics_config=None, prediction_intervals=None, predict_intervals=None, ensemble_config=None, trained_transformation_pipeline=None, transformed_X=None, transformed_y=None, return_transformation_pipeline=False, X_test_already_transformed=False, skip_feature_responses=None, prediction_interval_params=None): - self.set_params_and_defaults(raw_training_data, user_input_func=user_input_func, optimize_final_model=optimize_final_model, write_gs_param_results_to_file=write_gs_param_results_to_file, perform_feature_selection=perform_feature_selection, verbose=verbose, X_test=X_test, y_test=y_test, ml_for_analytics=ml_for_analytics, take_log_of_y=take_log_of_y, model_names=model_names, perform_feature_scaling=perform_feature_scaling, calibrate_final_model=calibrate_final_model, _scorer=_scorer, scoring=scoring, verify_features=verify_features, training_params=training_params, grid_search_params=grid_search_params, compare_all_models=compare_all_models, cv=cv, feature_learning=feature_learning, fl_data=fl_data, optimize_feature_learning=False, train_uncertainty_model=train_uncertainty_model, uncertainty_data=uncertainty_data, uncertainty_delta=uncertainty_delta, uncertainty_delta_units=uncertainty_delta_units, calibrate_uncertainty=calibrate_uncertainty, uncertainty_calibration_settings=uncertainty_calibration_settings, uncertainty_calibration_data=uncertainty_calibration_data, uncertainty_delta_direction=uncertainty_delta_direction, prediction_intervals=prediction_intervals, predict_intervals=predict_intervals, ensemble_config=ensemble_config, trained_transformation_pipeline=trained_transformation_pipeline, transformed_X=transformed_X, transformed_y=transformed_y, return_transformation_pipeline=return_transformation_pipeline, X_test_already_transformed=X_test_already_transformed, skip_feature_responses=skip_feature_responses) + self.set_params_and_defaults(raw_training_data, user_input_func=user_input_func, optimize_final_model=optimize_final_model, write_gs_param_results_to_file=write_gs_param_results_to_file, perform_feature_selection=perform_feature_selection, verbose=verbose, X_test=X_test, y_test=y_test, ml_for_analytics=ml_for_analytics, take_log_of_y=take_log_of_y, model_names=model_names, perform_feature_scaling=perform_feature_scaling, calibrate_final_model=calibrate_final_model, _scorer=_scorer, scoring=scoring, verify_features=verify_features, training_params=training_params, grid_search_params=grid_search_params, compare_all_models=compare_all_models, cv=cv, feature_learning=feature_learning, fl_data=fl_data, optimize_feature_learning=False, train_uncertainty_model=train_uncertainty_model, uncertainty_data=uncertainty_data, uncertainty_delta=uncertainty_delta, uncertainty_delta_units=uncertainty_delta_units, calibrate_uncertainty=calibrate_uncertainty, uncertainty_calibration_settings=uncertainty_calibration_settings, uncertainty_calibration_data=uncertainty_calibration_data, uncertainty_delta_direction=uncertainty_delta_direction, prediction_intervals=prediction_intervals, predict_intervals=predict_intervals, ensemble_config=ensemble_config, trained_transformation_pipeline=trained_transformation_pipeline, transformed_X=transformed_X, transformed_y=transformed_y, return_transformation_pipeline=return_transformation_pipeline, X_test_already_transformed=X_test_already_transformed, skip_feature_responses=skip_feature_responses, prediction_interval_params=prediction_interval_params) if verbose: print('Welcome to auto_ml! We\'re about to go through and make sense of your data using machine learning, and give you a production-ready pipeline to get predictions with.\n') @@ -647,7 +654,7 @@ def train(self, raw_training_data, user_input_func=None, optimize_final_model=No # TODO: parallelize these! interval_predictors = [] for percentile in self.prediction_intervals: - interval_predictor = self.train_ml_estimator(['LGBMRegressor'], self._scorer, X_df, y, prediction_interval=percentile) + interval_predictor = self.train_ml_estimator(['GradientBoostingRegressor'], self._scorer, X_df, y, prediction_interval=percentile) predictor_tup = ('interval_{}'.format(percentile), interval_predictor) interval_predictors.append(predictor_tup) @@ -1189,7 +1196,7 @@ def create_gs_params(self, model_name): def train_ml_estimator(self, estimator_names, scoring, X_df, y, feature_learning=False, prediction_interval=False): if prediction_interval is not False: - estimator_names = ['LGBMRegressor'] + # estimator_names = ['GradientBoostingRegressor'] trained_final_model = self.fit_single_pipeline(X_df, y, estimator_names[0], feature_learning=feature_learning, prediction_interval=prediction_interval) # Use Case 1: Super straightforward: just train a single, non-optimized model diff --git a/tests/core_tests/test_prediction_intervals.py b/tests/core_tests/test_prediction_intervals.py index 5484b11..8ffe9fd 100644 --- a/tests/core_tests/test_prediction_intervals.py +++ b/tests/core_tests/test_prediction_intervals.py @@ -28,8 +28,6 @@ def test_predict_uncertainty_true(): , 'CHAS': 'categorical' } - df_boston_train, uncertainty_data = train_test_split(df_boston_train, test_size=0.5) - ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) ml_predictor.train(df_boston_train, predict_intervals=True) @@ -81,11 +79,9 @@ def test_prediction_intervals_actually_work(): , 'CHAS': 'categorical' } - df_boston_train, uncertainty_data = train_test_split(df_boston_train, test_size=0.5) - ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) - ml_predictor.train(df_boston_train, predict_intervals=[0.1, 0.9]) + ml_predictor.train(df_boston_train, predict_intervals=[0.05, 0.95]) df_boston_test = df_boston_test.reset_index(drop=True) intervals = ml_predictor.predict_intervals(df_boston_test) @@ -93,13 +89,13 @@ def test_prediction_intervals_actually_work(): count_under = 0 count_over = 0 - print(intervals) + # print(intervals) for idx, row in intervals.iterrows(): actual = actuals.iloc[idx] - if actual < row['interval_0.1']: + if actual < row['interval_0.05']: count_under += 1 - if actual > row['interval_0.9']: + if actual > row['interval_0.95']: count_over += 1 len_intervals = len(intervals) @@ -107,8 +103,8 @@ def test_prediction_intervals_actually_work(): pct_under = count_under * 1.0 / len_intervals pct_over = count_over * 1.0 / len_intervals # There's a decent bit of noise since this is such a small dataset - assert pct_under < 0.3 - assert pct_over < 0.3 + assert pct_under < 0.15 + assert pct_over < 0.1 def test_prediction_intervals_lets_the_user_specify_number_of_intervals(): @@ -121,8 +117,6 @@ def test_prediction_intervals_lets_the_user_specify_number_of_intervals(): , 'CHAS': 'categorical' } - df_boston_train, uncertainty_data = train_test_split(df_boston_train, test_size=0.5) - ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) ml_predictor.train(df_boston_train, predict_intervals=True, prediction_intervals=[.2]) @@ -142,8 +136,6 @@ def test_predict_intervals_should_fail_if_not_trained(): , 'CHAS': 'categorical' } - df_boston_train, uncertainty_data = train_test_split(df_boston_train, test_size=0.5) - ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) ml_predictor.train(df_boston_train) @@ -167,10 +159,10 @@ def test_predict_intervals_takes_in_custom_intervals(): , 'CHAS': 'categorical' } - df_boston_train, uncertainty_data = train_test_split(df_boston_train, test_size=0.5) - ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) + # df_boston_train = pd.concat([df_boston_train, df_boston_train, df_boston_train]) + ml_predictor.train(df_boston_train, predict_intervals=[0.4, 0.6]) custom_intervals = ml_predictor.predict_intervals(df_boston_test, return_type='list') @@ -207,6 +199,7 @@ def test_predict_intervals_takes_in_custom_intervals(): default_intervals = ml_predictor.predict_intervals(df_boston_test, return_type='list') # This is a super flaky test, because we've got such a small datasize, and we're trying to get distributions from it + len_intervals = len(custom_intervals) num_failures = 0 for idx, custom_row in enumerate(custom_intervals): default_row = default_intervals[idx] @@ -218,5 +211,4 @@ def test_predict_intervals_takes_in_custom_intervals(): print('{} should be lower than {}'.format(custom_row[1], default_row[1])) num_failures += 1 - len_intervals = len(custom_intervals) - assert num_failures < 0.25 * len_intervals + assert num_failures < 0.18 * len_intervals