Remove default value of objective function and objective config field.

skll/config.py

@@ -72,8 +72,7 @@ def __init__(self):
                     'models': '',
                     'num_cv_folds': '10',
                     'metrics': "[]",
-                    'objectives': "['f1_score_micro']",
-                    'objective': "f1_score_micro",
+                    'objectives': "[]",
                     'param_grids': '[]',
                     'pos_label_str': '',
                     'predictions': '',
@@ -115,7 +114,6 @@ def __init__(self):
                                    'models': 'Output',
                                    'num_cv_folds': 'Input',
                                    'objectives': 'Tuning',
-                                   'objective': 'Tuning',
                                    'param_grids': 'Tuning',
                                    'pos_label_str': 'Tuning',
                                    'predictions': 'Output',
@@ -293,10 +291,6 @@ def _setup_config_parser(config_path, validate=True):
     ------
     IOError
         If the configuration file does not exist.
-    ValueError
-        If the configuration file specifies both objective and objectives.
-    TypeError
-        If any objective is not a string.
     """
     # initialize config parser with the given defaults
     config = SKLLConfigParser()
@@ -307,31 +301,6 @@ def _setup_config_parser(config_path, validate=True):
                       config_path)
     config.read(config_path)
 
-    # normalize objective to objectives
-    objective_value = config.get('Tuning', 'objective')
-    objectives_value = config.get('Tuning', 'objectives')
-    objective_default = config._defaults['objective']
-    objectives_default = config._defaults['objectives']
-
-    # if both of them are non default, raise error
-    if (objectives_value != objectives_default and objective_value != objective_default):
-        raise ValueError("The configuration file can specify "
-                         "either 'objective' or 'objectives', "
-                         "not both")
-    else:
-        # if objective is default value, delete it
-        if objective_value == objective_default:
-            config.remove_option('Tuning', 'objective')
-        else:
-            # else convert objective into objectives and delete objective
-            objective_value = yaml.safe_load(_fix_json(objective_value), )
-            if isinstance(objective_value, string_types):
-                config.set(
-                    'Tuning', 'objectives', "['{}']".format(objective_value))
-                config.remove_option('Tuning', 'objective')
-            else:
-                raise TypeError("objective should be a string")
-
     if validate:
         config.validate()
 
@@ -390,7 +359,7 @@ def _parse_config_file(config_path, log_level=logging.INFO):
     do_grid_search : bool
         Whether to perform grid search.
     grid_objectives : list of str
-        A list of objects functions to use for tuning.
+        A list of scoring functions to use for tuning.
     probability : bool
         Whether to output probabilities for each class.
     results_path : str
@@ -471,7 +440,9 @@ def _parse_config_file(config_path, log_level=logging.INFO):
 
     # extract parameters from the various sections in the config file
 
-    # 1. General
+    ######################
+    # 1. General section #
+    ######################
     if config.has_option("General", "experiment_name"):
         experiment_name = config.get("General", "experiment_name")
     else:
@@ -489,7 +460,7 @@ def _parse_config_file(config_path, log_level=logging.INFO):
             os.makedirs(log_path)
 
     # Create a top-level log file under the log path
-    main_log_file =join(log_path, '{}.log'.format(experiment_name))
+    main_log_file = join(log_path, '{}.log'.format(experiment_name))
 
     # Now create a SKLL logger that will log to this file as well
     # as to the console. Use the log level provided - note that
@@ -508,7 +479,9 @@ def _parse_config_file(config_path, log_level=logging.INFO):
         raise ValueError('An invalid task was specified: {}.  Valid tasks are:'
                          ' {}'.format(task, ', '.join(_VALID_TASKS)))
 
-    # 2. Input
+    ####################
+    # 2. Input section #
+    ####################
     sampler = config.get("Input", "sampler")
     if sampler not in _VALID_SAMPLERS:
         raise ValueError('An invalid sampler was specified: {}.  Valid '
@@ -585,8 +558,8 @@ def _parse_config_file(config_path, log_level=logging.INFO):
         learning_curve_cv_folds_list = [10] * len(learners)
     else:
         if (not isinstance(learning_curve_cv_folds_list, list) or
-            not all([isinstance(fold, int) for fold in learning_curve_cv_folds_list]) or
-            not len(learning_curve_cv_folds_list) == len(learners)):
+                not all([isinstance(fold, int) for fold in learning_curve_cv_folds_list]) or
+                not len(learning_curve_cv_folds_list) == len(learners)):
             raise ValueError("The learning_curve_cv_folds parameter should "
                              "be a list of integers of the same length as "
                              "the number of learners. You specified: {}"
@@ -603,7 +576,7 @@ def _parse_config_file(config_path, log_level=logging.INFO):
     else:
         if (not isinstance(learning_curve_train_sizes, list) or
             not all([isinstance(size, int) or isinstance(size, float) for size in
-                         learning_curve_train_sizes])):
+                     learning_curve_train_sizes])):
             raise ValueError("The learning_curve_train_sizes parameter should "
                              "be a list of integers or floats. You specified: {}"
                              .format(learning_curve_train_sizes))
@@ -728,7 +701,9 @@ def _parse_config_file(config_path, log_level=logging.INFO):
     else:
         class_map = None
 
-    # 3. Output
+    #####################
+    # 3. Output section #
+    #####################
     probability = config.getboolean("Output", "probability")
 
     # do we want to keep the predictions?
@@ -760,15 +735,24 @@ def _parse_config_file(config_path, log_level=logging.INFO):
     # what are the output metrics?
     output_metrics = config.get("Output", "metrics")
     output_metrics = _parse_and_validate_metrics(output_metrics,
-                                                     'metrics',
-                                                     logger=logger)
+                                                 'metrics',
+                                                 logger=logger)
+
+    #####################
+    # 4. Tuning section #
+    #####################
 
-    # 4. Tuning
     # do we need to run a grid search for the hyperparameters or are we just
     # using the defaults?
     do_grid_search = config.getboolean("Tuning", "grid_search")
 
-    # Check if `param_grids` is specified, but `grid_search` is False
+    # parse any provided grid objective functions
+    grid_objectives = config.get("Tuning", "objectives")
+    grid_objectives = _parse_and_validate_metrics(grid_objectives,
+                                                  'objectives',
+                                                  logger=logger)
+
+    # Check if `param_grids` is specified, but `do_grid_search` is False
     if param_grid_list and not do_grid_search:
         logger.warning('Since "grid_search" is set to False, the specified'
                        ' "param_grids" will be ignored.')
@@ -800,16 +784,17 @@ def _parse_config_file(config_path, log_level=logging.INFO):
     # how many folds should we run in parallel for grid search
     grid_search_folds = config.getint("Tuning", "grid_search_folds")
 
-    # what are the objective functions for the grid search?
-    grid_objectives = config.get("Tuning", "objectives")
-    grid_objectives = _parse_and_validate_metrics(grid_objectives,
-                                                  'objectives',
-                                                  logger=logger)
-
     # check whether the right things are set for the given task
     if (task == 'evaluate' or task == 'predict') and not test_path:
         raise ValueError('The test set must be set when task is evaluate or '
                          'predict.')
+    if task in ['cross_validate', 'evaluate', 'train']:
+        if do_grid_search and len(grid_objectives) == 0:
+            raise ValueError('You must specify a list of objectives if doing grid search.')
+        if not do_grid_search and len(grid_objectives) > 0:
+            logger.warning('Since "grid_search" is set to False, any specified'
+                           ' "objectives" will be ignored.')
+            grid_objectives = []
     if task in ['cross_validate', 'train', 'learning_curve'] and test_path:
         raise ValueError('The test set should not be set when task is '
                          '{}.'.format(task))
@@ -846,7 +831,7 @@ def _parse_config_file(config_path, log_level=logging.INFO):
         # some overlaps - we will assume that the user meant to
         # use the metric for tuning _and_ evaluation, not just evaluation
         if (len(grid_objectives) > 0 and
-            len(output_metrics) > 0):
+                len(output_metrics) > 0):
             common_metrics_and_objectives = set(grid_objectives).intersection(output_metrics)
             if common_metrics_and_objectives:
                 logger.warning('The following are specified both as '
@@ -990,7 +975,8 @@ def _parse_and_validate_metrics(metrics, option_name, logger=None):
     if not logger:
         logger = logging.getLogger(__name__)
 
-    # what are the objective functions for the grid search?
+    # make sure the given metrics data type is a list
+    # and parse it correctly
     metrics = yaml.safe_load(_fix_json(metrics))
     if not isinstance(metrics, list):
         raise TypeError("{} should be a list".format(option_name))

skll/experiments.py

@@ -251,7 +251,7 @@ def _write_learning_curve_file(result_json_paths, output_file):
         train_scores_stds_by_size = lrd['learning_curve_train_scores_stds']
         test_scores_stds_by_size = lrd['learning_curve_test_scores_stds']
 
-        # rename `grid_objective` to `objective` since that can be confusing
+        # rename `grid_objective` to `metric` since the latter name can be confusing
         lrd['metric'] = lrd['grid_objective']
 
         for (size,
@@ -321,8 +321,8 @@ def _print_fancy_output(learner_result_dicts, output_file=sys.stdout):
         print('Grid Objective Function: {}'.format(lrd['grid_objective']),
               file=output_file)
     if (lrd['task'] == 'cross_validate' and
-        lrd['grid_search'] and
-        lrd['cv_folds'].endswith('folds file')):
+            lrd['grid_search'] and
+            lrd['cv_folds'].endswith('folds file')):
         print('Using Folds File for Grid Search: {}'.format(lrd['use_folds_file_for_grid_search']),
               file=output_file)
     if lrd['task'] in ['evaluate', 'cross_validate'] and lrd['additional_scores']:
@@ -358,7 +358,7 @@ def _print_fancy_output(learner_result_dicts, output_file=sys.stdout):
                       file=output_file)
             print('Pearson = {: f}'.format(lrd['pearson']),
                   file=output_file)
-        print('Objective Function Score (Test) = {}'.format(lrd['score']),
+        print('Objective Function Score (Test) = {}'.format(lrd.get('score', '')),
               file=output_file)
 
         # now print the additional metrics, if there were any
@@ -580,8 +580,8 @@ def _classify_featureset(args):
     # featureset already exists if so, load it and then use it on test data
     modelfile = join(model_path, '{}.model'.format(job_name))
     if (task in ['cross_validate', 'learning_curve'] or
-        not exists(modelfile) or
-        overwrite):
+            not exists(modelfile) or
+            overwrite):
         train_examples = _load_featureset(train_path,
                                           featureset,
                                           suffix,
@@ -651,7 +651,6 @@ def _classify_featureset(args):
     else:
         grid_search_folds_to_print = str(grid_search_folds)
 
-
     # create a list of dictionaries of the results information
     learner_result_dict_base = {'experiment_name': experiment_name,
                                 'train_set_name': train_set_name,
@@ -674,8 +673,7 @@ def _classify_featureset(args):
                                 'grid_search_folds': grid_search_folds_to_print,
                                 'min_feature_count': min_feature_count,
                                 'cv_folds': cv_folds_to_print,
-                                'using_folds_file': isinstance(cv_folds, dict) \
-                                                     or isinstance(grid_search_folds, dict),
+                                'using_folds_file': isinstance(cv_folds, dict) or isinstance(grid_search_folds, dict),
                                 'save_cv_folds': save_cv_folds,
                                 'use_folds_file_for_grid_search': use_folds_file_for_grid_search,
                                 'stratified_folds': stratified_folds,
@@ -706,9 +704,9 @@ def _classify_featureset(args):
         (curve_train_scores,
          curve_test_scores,
          computed_curve_train_sizes) = learner.learning_curve(train_examples,
+                                                              grid_objective,
                                                               cv_folds=learning_curve_cv_folds,
-                                                              train_sizes=learning_curve_train_sizes,
-                                                              metric=grid_objective)
+                                                              train_sizes=learning_curve_train_sizes)
     else:
         # if we have do not have a saved model, we need to train one.
         if not exists(modelfile) or overwrite:
@@ -1034,7 +1032,7 @@ def run_configuration(config_file, local=False, overwrite=True, queue='all.q',
      param_grid_list, featureset_names, learners, prediction_dir, log_path, train_path,
      test_path, ids_to_floats, class_map, custom_learner_path, learning_curve_cv_folds_list,
      learning_curve_train_sizes, output_metrics) = _parse_config_file(config_file,
-                                                                          log_level=log_level)
+                                                                      log_level=log_level)
 
     # get the main experiment logger that will already have been
     # created by the configuration parser so we don't need anything
@@ -1131,14 +1129,21 @@ def run_configuration(config_file, local=False, overwrite=True, queue='all.q',
     if task == 'learning_curve' and len(output_metrics) > 0:
         grid_objectives = output_metrics
 
+    # if there were no grid objectives provided, just set it to
+    # a list containing a single None so as to allow the parallelization
+    # to proceeed and to pass the correct default value of grid_objective
+    # down to _classify_featureset().
+    if not grid_objectives:
+        grid_objectives = [None]
+
     # Run each featureset-learner-objective combination
     for featureset, featureset_name in zip(featuresets, featureset_names):
         for learner_num, learner_name in enumerate(learners):
             for grid_objective in grid_objectives:
 
                 # for the individual job name, we need to add the feature set name
                 # and the learner name
-                if len(grid_objectives) == 1:
+                if grid_objective is None or len(grid_objectives) == 1:
                     job_name_components = [experiment_name, featureset_name,
                                            learner_name]
                 else:
@@ -1424,4 +1429,4 @@ def _generate_learning_curve_plots(experiment_name,
                                       ncol=1,
                                       frameon=True)
             g.fig.tight_layout(w_pad=1)
-            plt.savefig(join(output_dir,'{}_{}.png'.format(experiment_name, fs_name)), dpi=300);
+            plt.savefig(join(output_dir, '{}_{}.png'.format(experiment_name, fs_name)), dpi=300);