Skip to content

Commit

Permalink
Comprehensive configs for trainer and combiner. (#2118)
Browse files Browse the repository at this point in the history
  • Loading branch information
@justinxzhao
justinxzhao committed Jun 10, 2022
1 parent ecc8efc commit ae25cc4
Show file tree
Hide file tree
Showing 4 changed files with 74 additions and 133 deletions.
16 changes: 9 additions & 7 deletions ludwig/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -420,11 +420,12 @@ def on_epoch_end(self, trainer, progress_tracker, save_path):
if key != "config": # Config is printed separately.
experiment_description.append([key, pformat(value, indent=4)])

print_boxed("EXPERIMENT DESCRIPTION")
logger.info(tabulate(experiment_description, tablefmt="fancy_grid"))
if self.backend.is_coordinator():
print_boxed("EXPERIMENT DESCRIPTION")
logger.info(tabulate(experiment_description, tablefmt="fancy_grid"))

print_boxed("LUDWIG CONFIG")
logger.info(pformat(self.config, indent=4))
print_boxed("LUDWIG CONFIG")
logger.info(pformat(self.config, indent=4))

for callback in self.callbacks:
callback.on_preprocess_start(self.config)
Expand Down Expand Up @@ -492,11 +493,12 @@ def on_epoch_end(self, trainer, progress_tracker, save_path):
logger.info("Warnings and other logs:")
self.model = LudwigModel.create_model(self.config, random_seed=random_seed)

# init trainer
config, _ = load_config_with_kwargs(Trainer.get_schema_cls(), self.config[TRAINER])
# Convert config dictionary into an instance of TrainerConfig.
trainer_config, _ = load_config_with_kwargs(Trainer.get_schema_cls(), self.config[TRAINER])

with self.backend.create_trainer(
model=self.model,
config=config,
config=trainer_config,
resume=model_resume_path is not None,
skip_save_model=skip_save_model,
skip_save_progress=skip_save_progress,
Expand Down
14 changes: 8 additions & 6 deletions ludwig/schema/utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from dataclasses import field
from typing import Dict as tDict
from typing import List, Tuple, Union
from typing import List, Tuple, Type, Union

from marshmallow import EXCLUDE, fields, schema, validate, ValidationError
from marshmallow_jsonschema import JSONSchema as js
Expand All @@ -9,20 +9,22 @@
from ludwig.utils.torch_utils import activations, initializer_registry


def load_config(cls, **kwargs):
def load_config(cls: Type["BaseMarshmallowConfig"], **kwargs): # noqa 0821
"""Takes a marshmallow class and instantiates it with the given keyword args as parameters."""
assert_is_a_marshmallow_class(cls)
schema = cls.Schema()
return schema.load(kwargs)


def load_config_with_kwargs(cls, kwargs):
"""Takes a marshmallow class and dict of parameter values and appropriately instantiantes the schema."""
def load_config_with_kwargs(
cls: Type["BaseMarshmallowConfig"], kwargs_overrides
) -> "BaseMarshmallowConfig": # noqa 0821
"""Instatiates an instance of the marshmallow class and kwargs overrides instantiantes the schema."""
assert_is_a_marshmallow_class(cls)
schema = cls.Schema()
fields = schema.fields.keys()
return load_config(cls, **{k: v for k, v in kwargs.items() if k in fields}), {
k: v for k, v in kwargs.items() if k not in fields
return load_config(cls, **{k: v for k, v in kwargs_overrides.items() if k in fields}), {
k: v for k, v in kwargs_overrides.items() if k not in fields
}


Expand Down
105 changes: 14 additions & 91 deletions ludwig/utils/defaults.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,18 @@
import copy
import logging
import sys
from dataclasses import asdict

import yaml

from ludwig.constants import (
BINARY,
CATEGORY,
COLUMN,
COMBINED,
COMBINER,
DROP_ROW,
EXECUTOR,
HYPEROPT,
LOSS,
NAME,
PREPROCESSING,
PROC_COLUMN,
Expand All @@ -40,6 +40,9 @@
from ludwig.features.feature_registries import base_type_registry, input_type_registry, output_type_registry
from ludwig.features.feature_utils import compute_feature_hash
from ludwig.globals import LUDWIG_VERSION
from ludwig.schema.combiners.utils import combiner_registry
from ludwig.schema.trainer import TrainerConfig
from ludwig.schema.utils import load_config_with_kwargs
from ludwig.utils.backward_compatibility import upgrade_deprecated_fields
from ludwig.utils.data_utils import load_config_from_str, load_yaml
from ludwig.utils.misc_utils import get_from_registry, merge_dict, set_default_value
Expand Down Expand Up @@ -70,82 +73,6 @@

default_combiner_type = "concat"

default_training_params = {
"optimizer": {TYPE: "adam"},
"epochs": 100,
"regularization_lambda": 0,
"regularization_type": "l2",
"learning_rate": 0.001,
"batch_size": 128,
"eval_batch_size": None,
"early_stop": 5,
"steps_per_checkpoint": 0,
"reduce_learning_rate_on_plateau": 0,
"reduce_learning_rate_on_plateau_patience": 5,
"reduce_learning_rate_on_plateau_rate": 0.5,
"increase_batch_size_on_plateau": 0,
"increase_batch_size_on_plateau_patience": 5,
"increase_batch_size_on_plateau_rate": 2,
"increase_batch_size_on_plateau_max": 512,
"decay": False,
"decay_steps": 10000,
"decay_rate": 0.96,
"staircase": False,
"gradient_clipping": None,
"validation_field": COMBINED,
"validation_metric": LOSS,
"learning_rate_warmup_epochs": 1,
}

default_optimizer_params_registry = {
"sgd": {},
"stochastic_gradient_descent": {},
"gd": {},
"gradient_descent": {},
"adam": {
"betas": (0.9, 0.999),
# 'beta_1': 0.9,
# 'beta_2': 0.999,
# 'epsilon': 1e-08
"eps": 1e-08,
},
"adamw": {
"betas": (0.9, 0.999),
"eps": 1e-08,
},
"adadelta": {
"rho": 0.95,
"eps": 1e-08
# 'epsilon': 1e-08
},
"adagrad": {"initial_accumulator_value": 0.1},
"adamax": {},
"ftrl": {
"learning_rate_power": -0.5,
"initial_accumulator_value": 0.1,
"l1_regularization_strength": 0.0,
"l2_regularization_strength": 0.0,
},
"nadam": {},
"rmsprop": {
"weight_decay": 0.9,
"momentum": 0.0,
# 'epsilon': 1e-10,
"eps": 1e-10,
"centered": False,
},
}
default_optimizer_params_registry["stochastic_gradient_descent"] = default_optimizer_params_registry["sgd"]
default_optimizer_params_registry["gd"] = default_optimizer_params_registry["sgd"]
default_optimizer_params_registry["gradient_descent"] = default_optimizer_params_registry["sgd"]


def get_default_optimizer_params(optimizer_type):
if optimizer_type in default_optimizer_params_registry:
return default_optimizer_params_registry[optimizer_type]
else:
raise ValueError("Incorrect optimizer type: " + optimizer_type)


def _perform_sanity_checks(config):
assert "input_features" in config, "config does not define any input features"
Expand Down Expand Up @@ -178,8 +105,8 @@ def _perform_sanity_checks(config):
"as a dictionary. Please check your config format."
)

if "combiner" in config:
assert isinstance(config["combiner"], dict), (
if COMBINER in config:
assert isinstance(config[COMBINER], dict), (
"There is an issue while reading the combiner section of the "
"config. The parameters are expected to be read"
"as a dictionary. Please check your config format."
Expand Down Expand Up @@ -260,29 +187,25 @@ def merge_with_defaults(config):
raise ValueError("Stratify feature must be binary or category")

# ===== Training =====
set_default_value(config, TRAINER, default_training_params)

for param, value in default_training_params.items():
set_default_value(config[TRAINER], param, value)
full_trainer_config, _ = load_config_with_kwargs(TrainerConfig, config[TRAINER] if TRAINER in config else {})
config[TRAINER] = asdict(full_trainer_config)

set_default_value(
config[TRAINER],
"validation_metric",
output_type_registry[config["output_features"][0][TYPE]].default_validation_metric,
)

# ===== Training Optimizer =====
optimizer = config[TRAINER]["optimizer"]
default_optimizer_params = get_default_optimizer_params(optimizer[TYPE])
for param in default_optimizer_params:
set_default_value(optimizer, param, default_optimizer_params[param])

# ===== Input Features =====
for input_feature in config["input_features"]:
get_from_registry(input_feature[TYPE], input_type_registry).populate_defaults(input_feature)

# ===== Combiner =====
set_default_value(config, "combiner", {TYPE: default_combiner_type})
set_default_value(config, COMBINER, {TYPE: default_combiner_type})
full_combiner_config, _ = load_config_with_kwargs(
combiner_registry[config[COMBINER][TYPE]].get_schema_cls(), config[COMBINER]
)
config[COMBINER].update(asdict(full_combiner_config))

# ===== Output features =====
for output_feature in config["output_features"]:
Expand Down
72 changes: 43 additions & 29 deletions tests/ludwig/utils/test_defaults.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
TYPE,
)
from ludwig.data.preprocessing import merge_preprocessing
from ludwig.utils.defaults import default_training_params, merge_with_defaults
from ludwig.schema.trainer import TrainerConfig
from ludwig.utils.defaults import merge_with_defaults
from tests.integration_tests.utils import (
binary_feature,
category_feature,
Expand Down Expand Up @@ -51,8 +52,6 @@

SCHEDULER_DICT = {"type": "async_hyperband", "time_attr": "time_total_s"}

default_early_stop = default_training_params["early_stop"]


@pytest.mark.parametrize(
"use_train,use_hyperopt_scheduler",
Expand Down Expand Up @@ -85,15 +84,15 @@ def test_merge_with_defaults_early_stop(use_train, use_hyperopt_scheduler):
config = copy.deepcopy(config)

if use_train:
config[TRAINER] = {"batch_size": "42"}
config[TRAINER] = {"batch_size": 42}

if use_hyperopt_scheduler:
# hyperopt scheduler cannot be used with early stopping
config[HYPEROPT]["executor"][SCHEDULER] = SCHEDULER_DICT

merged_config = merge_with_defaults(config)

expected = -1 if use_hyperopt_scheduler else default_early_stop
expected = -1 if use_hyperopt_scheduler else TrainerConfig().early_stop
assert merged_config[TRAINER]["early_stop"] == expected


Expand Down Expand Up @@ -194,7 +193,7 @@ def test_merge_with_defaults():
"name": "number_output_feature",
},
],
"training": {"eval_batch_size": 0},
"training": {"eval_batch_size": 0, "optimizer": {"type": "adadelta"}},
"hyperopt": {
"parameters": {
"training.learning_rate": {},
Expand Down Expand Up @@ -227,25 +226,16 @@ def test_merge_with_defaults():
{
"type": "image",
"name": "image_input_feature",
"column": "image_input_feature",
"preprocessing": {},
"proc_column": "image_input_feature_mZFLky",
"tied": None,
"encoder": "stacked_cnn",
"conv_use_bias": True,
"conv_layers": [
{
"num_filters": 32,
"pool_size": 2,
"pool_stride": 2,
"use_bias": False,
},
{
"num_filters": 64,
"pool_size": 2,
"pool_stride": 2,
},
{"num_filters": 32, "pool_size": 2, "pool_stride": 2, "use_bias": False},
{"num_filters": 64, "pool_size": 2, "pool_stride": 2},
],
"conv_use_bias": True,
"column": "image_input_feature",
"proc_column": "image_input_feature_mZFLky",
"tied": None,
"preprocessing": {},
},
],
"output_features": [
Expand Down Expand Up @@ -275,29 +265,38 @@ def test_merge_with_defaults():
},
"trainer": {
"eval_batch_size": None,
"optimizer": {"type": "adam", "betas": (0.9, 0.999), "eps": 1e-08},
"optimizer": {"type": "adadelta", "rho": 0.9, "eps": 1e-06, "lr": 1.0, "weight_decay": 0.0},
"epochs": 100,
"regularization_lambda": 0,
"train_steps": None,
"regularization_lambda": 0.0,
"regularization_type": "l2",
"should_shuffle": True,
"learning_rate": 0.001,
"batch_size": 128,
"early_stop": 5,
"steps_per_checkpoint": 0,
"reduce_learning_rate_on_plateau": 0,
"checkpoints_per_epoch": 0,
"evaluate_training_set": True,
"reduce_learning_rate_on_plateau": 0.0,
"reduce_learning_rate_on_plateau_patience": 5,
"reduce_learning_rate_on_plateau_rate": 0.5,
"reduce_learning_rate_eval_metric": "loss",
"reduce_learning_rate_eval_split": "training",
"increase_batch_size_on_plateau": 0,
"increase_batch_size_on_plateau_patience": 5,
"increase_batch_size_on_plateau_rate": 2,
"increase_batch_size_on_plateau_rate": 2.0,
"increase_batch_size_on_plateau_max": 512,
"increase_batch_size_eval_metric": "loss",
"increase_batch_size_eval_split": "training",
"decay": False,
"decay_steps": 10000,
"decay_rate": 0.96,
"staircase": False,
"gradient_clipping": None,
"gradient_clipping": {"clipglobalnorm": 0.5, "clipnorm": None, "clipvalue": None},
"validation_field": "combined",
"validation_metric": "loss",
"learning_rate_warmup_epochs": 1,
"learning_rate_warmup_epochs": 1.0,
"learning_rate_scaling": "linear",
},
"preprocessing": {
"force_split": False,
Expand Down Expand Up @@ -390,8 +389,23 @@ def test_merge_with_defaults():
"date": {"missing_value_strategy": "fill_with_const", "fill_value": "", "datetime_format": None},
"vector": {"missing_value_strategy": "fill_with_const", "fill_value": ""},
},
"combiner": {"type": "concat"},
"combiner": {
"type": "concat",
"fc_layers": None,
"num_fc_layers": 0,
"output_size": 256,
"use_bias": True,
"weights_initializer": "xavier_uniform",
"bias_initializer": "zeros",
"norm": None,
"norm_params": None,
"activation": "relu",
"dropout": 0.0,
"flatten_inputs": False,
"residual": False,
},
}

updated_config = merge_with_defaults(legacy_config_format)

assert updated_config == expected_upgraded_format

0 comments on commit ae25cc4

Please sign in to comment.