Skip to content

Commit

Permalink
Rename Environment.__init__.cross_validation_params to cv_params,…
Browse files Browse the repository at this point in the history
… but retain original as an `Alias`
  • Loading branch information
HunterMcGushion committed Feb 7, 2019
1 parent 998d7b9 commit f4ca93b
Show file tree
Hide file tree
Showing 46 changed files with 85 additions and 86 deletions.
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ env = Environment(
results_path='path/to/results/directory', # Where your result files will go
metrics_map=['roc_auc_score'], # Callables, or strings referring to `sklearn.metrics`
cross_validation_type=StratifiedKFold, # Class, or string in `sklearn.model_selection`
cross_validation_params=dict(n_splits=5, shuffle=True, random_state=32)
cv_params=dict(n_splits=5, shuffle=True, random_state=32)
)
```

Expand Down Expand Up @@ -358,7 +358,7 @@ env = Environment(
results_path='HyperparameterHunterAssets',
metrics_map=['roc_auc_score'],
cross_validation_type='StratifiedKFold',
cross_validation_params=dict(n_splits=10, shuffle=True, random_state=32),
cv_params=dict(n_splits=10, shuffle=True, random_state=32),
)

# Now, conduct an `Experiment`
Expand Down
2 changes: 1 addition & 1 deletion docs/quick_start.rst
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ Set Up an Environment
results_path="path/to/results/directory",
metrics_map=["roc_auc_score"],
cross_validation_type=StratifiedKFold,
cross_validation_params=dict(n_splits=5, shuffle=2, random_state=32)
cv_params=dict(n_splits=5, shuffle=2, random_state=32)
)
Individual Experimentation
Expand Down
2 changes: 1 addition & 1 deletion examples/advanced_examples/do_full_save_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def execute():
results_path="HyperparameterHunterAssets",
metrics_map=["roc_auc_score"],
cross_validation_type=RepeatedStratifiedKFold,
cross_validation_params=dict(n_splits=3, n_repeats=2, random_state=32),
cv_params=dict(n_splits=3, n_repeats=2, random_state=32),
do_full_save=do_full_save,
)

Expand Down
2 changes: 1 addition & 1 deletion examples/advanced_examples/environment_params.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"f1_score"
],
"cross_validation_type": "StratifiedKFold",
"cross_validation_params": {
"cv_params": {
"n_splits": 10,
"shuffle": true,
"random_state": 117
Expand Down
4 changes: 2 additions & 2 deletions examples/advanced_examples/environment_params_path_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def _execute():
env = Environment(
train_dataset=get_breast_cancer_data(), # If your dataset is a str path, you can even add it to environment_params
environment_params_path="./environment_params.json", # Use this file for parameters not explicitly given
cross_validation_params=dict(
cv_params=dict(
n_splits=5, shuffle=True, random_state=32
), # Here we decide to override our default values
)
Expand All @@ -30,7 +30,7 @@ def _execute():
print(env.file_blacklist) # This includes some other values too, but you can ignore them
# All of the above are from `environment_params_path`
print(
env.cross_validation_params
env.cv_params
) # This is the value we provided above, rather than our `environment_params_path` default

experiment = CVExperiment(model_initializer=KNeighborsClassifier, model_init_params={})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ def execute():
# See the documentation in :class:`environment.Environment` and :class:`metrics.ScoringMixIn` for more information
metrics_map=["roc_auc_score"],
cross_validation_type=StratifiedKFold,
cross_validation_params=dict(n_splits=5, shuffle=True, random_state=32),
cv_params=dict(n_splits=5, shuffle=True, random_state=32),
)

experiment = CVExperiment(
Expand Down
2 changes: 1 addition & 1 deletion examples/advanced_examples/lambda_callback_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def execute():
results_path="HyperparameterHunterAssets",
metrics_map=["roc_auc_score"],
cross_validation_type=RepeatedStratifiedKFold,
cross_validation_params=dict(n_splits=5, n_repeats=2, random_state=32),
cv_params=dict(n_splits=5, n_repeats=2, random_state=32),
runs=2,
# Just instantiate `Environment` with your list of callbacks, and go about business as usual
experiment_callbacks=[printer_callback(), confusion_matrix_oof()],
Expand Down
2 changes: 1 addition & 1 deletion examples/advanced_examples/multi_metric_example.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@
" f1_macro=lambda y_true, y_pred: f1_score(y_true, y_pred, average=\"macro\"),\n",
" ),\n",
" cross_validation_type=\"KFold\",\n",
" cross_validation_params=dict(n_splits=10, shuffle=True, random_state=42),\n",
" cv_params=dict(n_splits=10, shuffle=True, random_state=42),\n",
" verbose=1,\n",
")"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/advanced_examples/multi_metric_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
f1_macro=lambda y_true, y_pred: f1_score(y_true, y_pred, average="macro"),
),
cross_validation_type="KFold",
cross_validation_params=dict(n_splits=10, shuffle=True, random_state=42),
cv_params=dict(n_splits=10, shuffle=True, random_state=42),
verbose=1,
)

Expand Down
2 changes: 1 addition & 1 deletion examples/advanced_examples/recorder_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
target_column="diagnosis",
metrics_map=["roc_auc_score"],
cross_validation_type=StratifiedKFold,
cross_validation_params=dict(n_splits=10, shuffle=True, random_state=32),
cv_params=dict(n_splits=10, shuffle=True, random_state=32),
)


Expand Down
2 changes: 1 addition & 1 deletion examples/catboost_examples/classification.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@
" target_column=\"y\",\n",
" metrics_map=dict(f1=lambda y_true, y_pred: f1_score(y_true, y_pred, average=\"micro\")),\n",
" cross_validation_type=KFold,\n",
" cross_validation_params=dict(n_splits=5, random_state=32),\n",
" cv_params=dict(n_splits=5, random_state=32),\n",
")"
]
},
Expand Down
2 changes: 1 addition & 1 deletion examples/catboost_examples/classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
target_column="y",
metrics_map=dict(f1=lambda y_true, y_pred: f1_score(y_true, y_pred, average="micro")),
cross_validation_type=KFold,
cross_validation_params=dict(n_splits=5, random_state=32),
cv_params=dict(n_splits=5, random_state=32),
)

# Now that HyperparameterHunter has an active `Environment`, we can do two things:
Expand Down
2 changes: 1 addition & 1 deletion examples/catboost_examples/regression.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@
" results_path=\"HyperparameterHunterAssets\",\n",
" metrics_map=dict(evs=explained_variance_score),\n",
" cross_validation_type=\"KFold\",\n",
" cross_validation_params=dict(n_splits=3, shuffle=True, random_state=1337),\n",
" cv_params=dict(n_splits=3, shuffle=True, random_state=1337),\n",
" runs=2,\n",
")"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/catboost_examples/regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
results_path="HyperparameterHunterAssets",
metrics_map=dict(evs=explained_variance_score),
cross_validation_type="KFold",
cross_validation_params=dict(n_splits=3, shuffle=True, random_state=1337),
cv_params=dict(n_splits=3, shuffle=True, random_state=1337),
runs=2,
)

Expand Down
4 changes: 2 additions & 2 deletions examples/extended_example.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@
" metrics_map=['roc_auc_score'],\n",
" target_column='diagnosis',\n",
" cross_validation_type=StratifiedKFold,\n",
" cross_validation_params=dict(n_splits=3, shuffle=True, random_state=32),\n",
" cv_params=dict(n_splits=3, shuffle=True, random_state=32),\n",
" file_blacklist=['script_backup']\n",
")"
]
Expand All @@ -96,7 +96,7 @@
"* A `metrics_map`, with 'roc_auc_score', because the Wisconsin Breast Cancer dataset is a classification problem,\n",
"* `target_column`, a string naming the column in our dataset containing the target output (defaults to 'target'),\n",
"* Our `cross_validation_type` imported earlier,\n",
"* `cross_validation_params`, a dict containing all the arguments to pass to `cross_validation_type`,\n",
"* `cv_params`, a dict containing all the arguments to pass to `cross_validation_type`,\n",
"* And `file_blacklist`, a list of result file names we don't want to save. Here, we're giving it 'script_backup' because we're executing our Experiments in a Jupyter notebook, instead of a standard Python script\n",
"\n",
"Notice that upon instantiation, our `Environment` logs the cross-experiment key produced by the provided parameters. This is important because it determines when two `Experiment`s can be properly compared; we'll go over this more later."
Expand Down
2 changes: 1 addition & 1 deletion examples/functionality_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def execute():
results_path="HyperparameterHunterAssets",
metrics_map=["roc_auc_score"],
cross_validation_type="StratifiedKFold",
cross_validation_params=dict(n_splits=10, shuffle=True, random_state=32),
cv_params=dict(n_splits=10, shuffle=True, random_state=32),
)

# Now, conduct an `Experiment`
Expand Down
2 changes: 1 addition & 1 deletion examples/keras_examples/experiment_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def execute():
target_column="diagnosis",
metrics_map=["roc_auc_score"],
cross_validation_type="StratifiedKFold",
cross_validation_params=dict(n_splits=5, shuffle=True, random_state=32),
cv_params=dict(n_splits=5, shuffle=True, random_state=32),
)

experiment = CVExperiment(
Expand Down
2 changes: 1 addition & 1 deletion examples/keras_examples/image_classification_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def _execute():
results_path="HyperparameterHunterAssets",
metrics_map=["roc_auc_score"],
cross_validation_type="StratifiedKFold",
cross_validation_params=dict(n_splits=3, shuffle=True, random_state=True),
cv_params=dict(n_splits=3, shuffle=True, random_state=True),
)

experiment = CVExperiment(
Expand Down
2 changes: 1 addition & 1 deletion examples/keras_examples/multi_classification_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def _execute():
metrics_map=["roc_auc_score"],
target_column=[f"target_{_}" for _ in range(10)],
cross_validation_type="StratifiedKFold",
cross_validation_params=dict(n_splits=10, shuffle=True, random_state=True),
cv_params=dict(n_splits=10, shuffle=True, random_state=True),
)

experiment = CVExperiment(
Expand Down
2 changes: 1 addition & 1 deletion examples/keras_examples/optimization_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def _execute():
results_path="HyperparameterHunterAssets",
metrics_map=["roc_auc_score"],
cross_validation_type="StratifiedKFold",
cross_validation_params=dict(n_splits=5, shuffle=True, random_state=32),
cv_params=dict(n_splits=5, shuffle=True, random_state=32),
)

#################### Experimentation ####################
Expand Down
2 changes: 1 addition & 1 deletion examples/lightgbm_examples/classification.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@
" target_column=\"y\",\n",
" metrics_map=dict(f1=lambda y_true, y_pred: f1_score(y_true, y_pred, average=\"micro\")),\n",
" cross_validation_type=\"StratifiedKFold\",\n",
" cross_validation_params=dict(n_splits=5, random_state=32),\n",
" cv_params=dict(n_splits=5, random_state=32),\n",
")"
]
},
Expand Down
2 changes: 1 addition & 1 deletion examples/lightgbm_examples/classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
target_column="y",
metrics_map=dict(f1=lambda y_true, y_pred: f1_score(y_true, y_pred, average="micro")),
cross_validation_type="StratifiedKFold",
cross_validation_params=dict(n_splits=5, random_state=32),
cv_params=dict(n_splits=5, random_state=32),
)

# Now that HyperparameterHunter has an active `Environment`, we can do two things:
Expand Down
2 changes: 1 addition & 1 deletion examples/lightgbm_examples/regression.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@
" target_column=\"median_value\",\n",
" metrics_map=dict(r2=r2_score),\n",
" cross_validation_type=RepeatedKFold,\n",
" cross_validation_params=dict(n_repeats=2, n_splits=5, random_state=42),\n",
" cv_params=dict(n_repeats=2, n_splits=5, random_state=42),\n",
")"
]
},
Expand Down
2 changes: 1 addition & 1 deletion examples/lightgbm_examples/regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
target_column="median_value",
metrics_map=dict(r2=r2_score),
cross_validation_type=RepeatedKFold,
cross_validation_params=dict(n_repeats=2, n_splits=5, random_state=42),
cv_params=dict(n_repeats=2, n_splits=5, random_state=42),
)

# Now that HyperparameterHunter has an active `Environment`, we can do two things:
Expand Down
2 changes: 1 addition & 1 deletion examples/rgf_examples/classification.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@
" target_column=\"y\",\n",
" metrics_map=[\"hamming_loss\"],\n",
" cross_validation_type=RepeatedStratifiedKFold,\n",
" cross_validation_params=dict(n_repeats=2, n_splits=10, random_state=1337),\n",
" cv_params=dict(n_repeats=2, n_splits=10, random_state=1337),\n",
")"
]
},
Expand Down
2 changes: 1 addition & 1 deletion examples/rgf_examples/classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
target_column="y",
metrics_map=["hamming_loss"],
cross_validation_type=RepeatedStratifiedKFold,
cross_validation_params=dict(n_repeats=2, n_splits=10, random_state=1337),
cv_params=dict(n_repeats=2, n_splits=10, random_state=1337),
)

# Now that HyperparameterHunter has an active `Environment`, we can do two things:
Expand Down
2 changes: 1 addition & 1 deletion examples/rgf_examples/regression.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@
" target_column=\"progression\",\n",
" metrics_map=dict(msle=(mean_squared_log_error, \"min\")),\n",
" cross_validation_type=\"KFold\",\n",
" cross_validation_params=dict(n_splits=10, random_state=42),\n",
" cv_params=dict(n_splits=10, random_state=42),\n",
" runs=3,\n",
")"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/rgf_examples/regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
target_column="progression",
metrics_map=dict(msle=(mean_squared_log_error, "min")),
cross_validation_type="KFold",
cross_validation_params=dict(n_splits=10, random_state=42),
cv_params=dict(n_splits=10, random_state=42),
runs=3,
)

Expand Down
2 changes: 1 addition & 1 deletion examples/simple_experiment_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ def execute():
results_path="HyperparameterHunterAssets",
metrics_map=["roc_auc_score"],
cross_validation_type="StratifiedKFold",
cross_validation_params=dict(n_splits=5, shuffle=True, random_state=32),
cv_params=dict(n_splits=5, shuffle=True, random_state=32),
)

experiment = CVExperiment(
Expand Down
2 changes: 1 addition & 1 deletion examples/simple_optimization_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ def _execute():
target_column="diagnosis",
metrics_map=["roc_auc_score"],
cross_validation_type=StratifiedKFold,
cross_validation_params=dict(n_splits=10, shuffle=True, random_state=32),
cv_params=dict(n_splits=10, shuffle=True, random_state=32),
runs=2,
)

Expand Down
2 changes: 1 addition & 1 deletion examples/sklearn_examples/classification.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@
" target_column=\"diagnosis\",\n",
" metrics_map=[\"roc_auc_score\"],\n",
" cross_validation_type=\"StratifiedKFold\",\n",
" cross_validation_params=dict(n_splits=5, random_state=32),\n",
" cv_params=dict(n_splits=5, random_state=32),\n",
" verbose=1,\n",
")"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/sklearn_examples/classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
target_column="diagnosis",
metrics_map=["roc_auc_score"],
cross_validation_type="StratifiedKFold",
cross_validation_params=dict(n_splits=5, random_state=32),
cv_params=dict(n_splits=5, random_state=32),
verbose=1,
)

Expand Down
2 changes: 1 addition & 1 deletion examples/xgboost_examples/classification.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@
" target_column=\"diagnosis\",\n",
" metrics_map=[\"roc_auc_score\"],\n",
" cross_validation_type=StratifiedKFold,\n",
" cross_validation_params=dict(n_splits=10, shuffle=True, random_state=32),\n",
" cv_params=dict(n_splits=10, shuffle=True, random_state=32),\n",
")"
]
},
Expand Down
2 changes: 1 addition & 1 deletion examples/xgboost_examples/classification.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
target_column="diagnosis",
metrics_map=["roc_auc_score"],
cross_validation_type=StratifiedKFold,
cross_validation_params=dict(n_splits=10, shuffle=True, random_state=32),
cv_params=dict(n_splits=10, shuffle=True, random_state=32),
)

# Now that HyperparameterHunter has an active `Environment`, we can do two things:
Expand Down
2 changes: 1 addition & 1 deletion examples/xgboost_examples/regression.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@
" target_column=\"progression\",\n",
" metrics_map=[\"mean_absolute_error\"],\n",
" cross_validation_type=\"KFold\",\n",
" cross_validation_params=dict(n_splits=12, shuffle=True, random_state=32),\n",
" cv_params=dict(n_splits=12, shuffle=True, random_state=32),\n",
" runs=2,\n",
")"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/xgboost_examples/regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
target_column="progression",
metrics_map=["mean_absolute_error"],
cross_validation_type="KFold",
cross_validation_params=dict(n_splits=12, shuffle=True, random_state=32),
cv_params=dict(n_splits=12, shuffle=True, random_state=32),
runs=2,
)

Expand Down
6 changes: 3 additions & 3 deletions hyperparameter_hunter/callbacks/loggers.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def on_experiment_start(self):
super().on_experiment_start()

def on_repetition_start(self):
if G.Env.verbose >= 3 and G.Env.cross_validation_params.get("n_repeats", 1) > 1:
if G.Env.verbose >= 3 and G.Env.cv_params.get("n_repeats", 1) > 1:
G.log("", previous_frame=inspect.currentframe().f_back)
super().on_repetition_start()

Expand Down Expand Up @@ -77,7 +77,7 @@ def on_fold_end(self):
content += self.log_separator if not content.endswith(" ") else ""
content += self.__elapsed_helper("folds")

if G.Env.verbose >= 2 and G.Env.cross_validation_params["n_splits"] > 1:
if G.Env.verbose >= 2 and G.Env.cv_params["n_splits"] > 1:
G.log(content, previous_frame=inspect.currentframe().f_back, add_time=False)
else:
G.debug(content, previous_frame=inspect.currentframe().f_back, add_time=False)
Expand All @@ -90,7 +90,7 @@ def on_repetition_end(self):
content += self.log_separator if not content.endswith(" ") else ""
content += self.__elapsed_helper("reps")

if G.Env.verbose >= 2 and G.Env.cross_validation_params.get("n_repeats", 1) > 1:
if G.Env.verbose >= 2 and G.Env.cv_params.get("n_repeats", 1) > 1:
G.log(content, previous_frame=inspect.currentframe().f_back)
else:
G.debug(content, previous_frame=inspect.currentframe().f_back)
Expand Down

0 comments on commit f4ca93b

Please sign in to comment.