Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Changes batch norm momentum defaults to 1-momentum #2100

Merged
merged 2 commits into from
Jun 7, 2022
Merged

Changes batch norm momentum defaults to 1-momentum #2100

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b7ec3ec
Changes batch norm momentum to 1-momentum.
dantreiman Jun 6, 2022
592db73
Updates schema description for bn_momentum.
dantreiman Jun 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/tabnet/higgs/medium_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ combiner:
output_size: 96 # N_d
sparsity: 0.000001 # lambda_sparse
bn_virtual_divider: 32 # factor to divide batch_size B to get B_v from the paper
bn_momentum: 0.9 # m_B
bn_momentum: 0.1 # m_B
num_steps: 8 # N_steps
relaxation_factor: 2 # gamma
bn_virtual_bs: 256 # B_v
Expand Down
2 changes: 1 addition & 1 deletion examples/tabnet/higgs/small_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ combiner:
output_size: 26 # N_d
sparsity: 0.000001 # lambda_sparse
bn_virtual_divider: 32 # factor to divide batch_size B to get B_v from the paper
bn_momentum: 0.6 # m_B
bn_momentum: 0.4 # m_B
num_steps: 5 # N_steps
relaxation_factor: 1.5 # gamma
bn_virtual_bs: 512 # B_v
Expand Down
2 changes: 1 addition & 1 deletion ludwig/automl/defaults/combiner/tabnet_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,4 +51,4 @@ hyperopt:
categories: [256, 512, 1024, 2048, 4096]
combiner.bn_momentum:
space: choice
categories: [0.6, 0.7, 0.8, 0.9, 0.95, 0.98]
categories: [0.4, 0.3, 0.2, 0.1, 0.05, 0.02]
24 changes: 12 additions & 12 deletions ludwig/automl/defaults/reference_configs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ datasets:
size: 8 # N_a
output_size: 128 # N_d
sparsity: 0.0 # lambda_sparse
bn_momentum: 0.6 # m_B
bn_momentum: 0.4 # m_B
num_steps: 3 # N_steps
relaxation_factor: 1.0 # gamma
bn_virtual_bs: 4096 # B_v
Expand Down Expand Up @@ -469,7 +469,7 @@ datasets:
size: 128 # N_a
output_size: 8 # N_d
sparsity: 0.0 # lambda_sparse
bn_momentum: 0.98 # m_B
bn_momentum: 0.02 # m_B
num_steps: 10 # N_steps
relaxation_factor: 1.0 # gamma
bn_virtual_bs: 4096 # B_v
Expand Down Expand Up @@ -764,7 +764,7 @@ datasets:
size: 32 # N_a
output_size: 8 # N_d
sparsity: 0.0 # lambda_sparse
bn_momentum: 0.98 # m_B
bn_momentum: 0.02 # m_B
num_steps: 3 # N_steps
relaxation_factor: 1.0 # gamma
bn_virtual_bs: 256 # B_v
Expand Down Expand Up @@ -1661,7 +1661,7 @@ datasets:
size: 128 # N_a
output_size: 24 # N_d
sparsity: 0.000001 # lambda_sparse
bn_momentum: 0.98 # m_B
bn_momentum: 0.02 # m_B
num_steps: 10 # N_steps
relaxation_factor: 1.0 # gamma
bn_virtual_bs: 2048 # B_v
Expand Down Expand Up @@ -2446,7 +2446,7 @@ datasets:
size: 128 # N_a
output_size: 8 # N_d
sparsity: 0.1 # lambda_sparse
bn_momentum: 0.9 # m_B
bn_momentum: 0.1 # m_B
num_steps: 9 # N_steps
relaxation_factor: 1.0 # gamma
bn_virtual_bs: 256 # B_v
Expand Down Expand Up @@ -2665,7 +2665,7 @@ datasets:
size: 128 # N_a
output_size: 128 # N_d
sparsity: 0.0 # lambda_sparse
bn_momentum: 0.8 # m_B
bn_momentum: 0.2 # m_B
num_steps: 3 # N_steps
relaxation_factor: 1.0 # gamma
bn_virtual_bs: 512 # B_v
Expand Down Expand Up @@ -2718,7 +2718,7 @@ datasets:
size: 16 # N_a
output_size: 128 # N_d
sparsity: 0.0 # lambda_sparse
bn_momentum: 0.98 # m_B
bn_momentum: 0.02 # m_B
num_steps: 6 # N_steps
relaxation_factor: 1.0 # gamma
bn_virtual_bs: 512 # B_v
Expand Down Expand Up @@ -2865,7 +2865,7 @@ datasets:
size: 32 # N_a
output_size: 32 # N_d
sparsity: 0.0001 # lambda_sparse
bn_momentum: 0.6 # m_B
bn_momentum: 0.4 # m_B
num_steps: 5 # N_steps
relaxation_factor: 1.2 # gamma
bn_virtual_bs: 1024 # B_v
Expand Down Expand Up @@ -3636,7 +3636,7 @@ datasets:
size: 24 # N_a
output_size: 128 # N_d
sparsity: 0.001 # lambda_sparse
bn_momentum: 0.8 # m_B
bn_momentum: 0.2 # m_B
num_steps: 7 # N_steps
relaxation_factor: 1.2 # gamma
bn_virtual_bs: 256 # B_v
Expand Down Expand Up @@ -4069,7 +4069,7 @@ datasets:
size: 8 # N_a
output_size: 8 # N_d
sparsity: 0.0 # lambda_sparse
bn_momentum: 0.6 # m_B
bn_momentum: 0.4 # m_B
num_steps: 3 # N_steps
relaxation_factor: 2.0 # gamma
bn_virtual_bs: 256 # B_v
Expand Down Expand Up @@ -4144,7 +4144,7 @@ datasets:
size: 128 # N_a
output_size: 8 # N_d
sparsity: 0.000001 # lambda_sparse
bn_momentum: 0.98 # m_B
bn_momentum: 0.02 # m_B
num_steps: 4 # N_steps
relaxation_factor: 1.2 # gamma
bn_virtual_bs: 4096 # B_v
Expand Down Expand Up @@ -4187,7 +4187,7 @@ datasets:
size: 32 # N_a
output_size: 128 # N_d
sparsity: 0.000001 # lambda_sparse
bn_momentum: 0.6 # m_B
bn_momentum: 0.4 # m_B
num_steps: 4 # N_steps
relaxation_factor: 1.2 # gamma
bn_virtual_bs: 4096 # B_v
Expand Down
2 changes: 1 addition & 1 deletion ludwig/encoders/image_encoders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ def __init__(
conv_stride: Union[int, Tuple[int]] = 1,
first_pool_kernel_size: Union[int, Tuple[int]] = None,
first_pool_stride: Union[int, Tuple[int]] = None,
batch_norm_momentum: float = 0.9,
batch_norm_momentum: float = 0.1,
batch_norm_epsilon: float = 0.001,
fc_layers: Optional[List[Dict]] = None,
num_fc_layers: Optional[int] = 1,
Expand Down
8 changes: 4 additions & 4 deletions ludwig/modules/convolutional_modules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -859,7 +859,7 @@ def __init__(
first_in_channels: int,
out_channels: int,
stride: int = 1,
batch_norm_momentum: float = 0.9,
batch_norm_momentum: float = 0.1,
batch_norm_epsilon: float = 0.001,
projection_shortcut: Optional[LudwigModule] = None,
):
Expand Down Expand Up @@ -948,7 +948,7 @@ def __init__(
first_in_channels: int,
out_channels: int,
stride: int = 1,
batch_norm_momentum: float = 0.9,
batch_norm_momentum: float = 0.1,
batch_norm_epsilon: float = 0.001,
projection_shortcut: Optional[LudwigModule] = None,
):
Expand Down Expand Up @@ -1065,7 +1065,7 @@ def __init__(
block_fn: Union[ResNetBlock, ResNetBottleneckBlock],
num_blocks: int,
stride: Union[int, Tuple[int]] = 1,
batch_norm_momentum: float = 0.9,
batch_norm_momentum: float = 0.1,
batch_norm_epsilon: float = 0.001,
):
super().__init__()
Expand Down Expand Up @@ -1147,7 +1147,7 @@ def __init__(
first_pool_stride: Union[int, Tuple[int]] = 2,
block_sizes: List[int] = None,
block_strides: List[Union[int, Tuple[int]]] = None,
batch_norm_momentum: float = 0.9,
batch_norm_momentum: float = 0.1,
batch_norm_epsilon: float = 0.001,
):
"""Creates a model obtaining an image representation.
Expand Down
2 changes: 1 addition & 1 deletion ludwig/modules/normalization_modules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
# implementation adapted from https://github.com/dreamquark-ai/tabnet
class GhostBatchNormalization(LudwigModule):
def __init__(
self, num_features: int, momentum: float = 0.9, epsilon: float = 1e-3, virtual_batch_size: Optional[int] = None
self, num_features: int, momentum: float = 0.1, epsilon: float = 1e-3, virtual_batch_size: Optional[int] = None
):
super().__init__()
self.num_features = num_features
Expand Down
8 changes: 4 additions & 4 deletions ludwig/modules/tabnet_modules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def __init__(
num_total_blocks: int = 4,
num_shared_blocks: int = 2,
relaxation_factor: float = 1.5,
bn_momentum: float = 0.7,
bn_momentum: float = 0.3,
bn_epsilon: float = 1e-3,
bn_virtual_bs: Optional[int] = None,
sparsity: float = 1e-5,
Expand Down Expand Up @@ -171,7 +171,7 @@ def __init__(
input_size: int,
size: int,
apply_glu: bool = True,
bn_momentum: float = 0.9,
bn_momentum: float = 0.1,
bn_epsilon: float = 1e-3,
bn_virtual_bs: int = None,
shared_fc_layer: LudwigModule = None,
Expand Down Expand Up @@ -215,7 +215,7 @@ def __init__(
self,
input_size: int,
size: int,
bn_momentum: float = 0.9,
bn_momentum: float = 0.1,
bn_epsilon: float = 1e-3,
bn_virtual_bs: int = None,
entmax_mode: str = "sparsemax",
Expand Down Expand Up @@ -285,7 +285,7 @@ def __init__(
shared_fc_layers: List = [],
num_total_blocks: int = 4,
num_shared_blocks: int = 2,
bn_momentum: float = 0.9,
bn_momentum: float = 0.1,
bn_epsilon: float = 1e-3,
bn_virtual_bs: int = None,
):
Expand Down
2 changes: 1 addition & 1 deletion ludwig/schema/combiners/tabnet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ class TabNetCombinerConfig(BaseCombinerConfig):
)

bn_momentum: float = schema_utils.FloatRange(
default=0.95, description="Momentum of the batch norm. `m_B` in the paper."
default=0.05, description="Momentum of the batch norm. `m_B` in the paper."
dantreiman marked this conversation as resolved.
Show resolved Hide resolved
)

bn_virtual_bs: Optional[int] = schema_utils.PositiveInteger(
Expand Down
2 changes: 1 addition & 1 deletion tests/ludwig/utils/test_validate_config_combiner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def test_config_tabnet(eval_batch_size):
"output_size": 26,
"sparsity": 0.000001,
"bn_virtual_divider": 32,
"bn_momentum": 0.6,
"bn_momentum": 0.4,
"num_steps": 5,
"relaxation_factor": 1.5,
"use_keras_batch_norm": False,
Expand Down
2 changes: 1 addition & 1 deletion tests/ludwig/utils/test_validate_config_misc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ def test_config_tabnet():
"output_size": 26,
"sparsity": 0.000001,
"bn_virtual_divider": 32,
"bn_momentum": 0.6,
"bn_momentum": 0.4,
"num_steps": 5,
"relaxation_factor": 1.5,
"bn_virtual_bs": 512,
Expand Down