feat: training assistant

bert_squeeze/assistants/__init__.py

@@ -0,0 +1 @@
+from .train_assistant import TrainAssistant

bert_squeeze/assistants/configs/train_bert.yaml

@@ -0,0 +1,47 @@
+general:
+  debug: false
+  do_train: true
+  do_eval: false
+  get_mismatched: true
+  evaluate_during_training: true
+  labels: [ 0, 1 ]
+  output_dir: outputs
+  save_steps: 500
+  validation_every_n_epoch: 1
+
+train:
+  adam_eps: 1e-8
+  accumulation_steps: 1
+  auto_lr: false
+  discriminative_learning: true
+  dropout: 0.2
+  layer_lr_decay: 0.95
+  learning_rates: [ 2e-5 ]
+  logging_steps: 50
+  lr_scheduler: true
+  max_grad_norm: 1.0
+  num_epochs: 10
+  optimizer: bertadam
+  objective: lsl
+  smoothing: 0.1
+  warmup_ratio: 0.06
+  warmup_steps: true
+  weight_decay: 0.01
+
+model:
+  _target_: bert_squeeze.models.lt_bert.LtCustomBert
+  num_labels: 2
+  pretrained_model: "bert-base-cased"
+  training_config: ${train}
+
+data:
+  _target_: bert_squeeze.data.modules.transformer_module.TransformerDataModule
+  dataset_config:
+    is_local: false
+    label_col: label
+    path:
+    split:
+    text_col: text
+    truncate_mode: head
+  max_length: 256
+  tokenizer_name: ${model.pretrained_model}

bert_squeeze/assistants/configs/train_deebert.yaml

@@ -0,0 +1,51 @@
+general:
+  debug: false
+  do_train: true
+  do_eval: false
+  get_mismatched: true
+  evaluate_during_training: true
+  labels: [ 0,1 ]
+  num_labels: 2
+  output_dir: outputs
+  save_steps: 500
+  validation_every_n_epoch: 1
+
+train:
+  adam_eps: 1e-8
+  accumulation_steps: 1
+  auto_lr: false
+  discriminative_learning: true
+  dropout: 0.2
+  layer_lr_decay: 0.95
+  learning_rates: [ 2e-5 ]
+  logging_steps: 100
+  lr_scheduler: true
+  max_grad_norm: 1.0
+  num_epochs: 10
+  optimizer: bertadam
+  objective: lsl
+  smoothing: 0.1
+  warmup_ratio: 0.06
+  warmup_steps: true
+  weight_decay: 0.01
+
+  train_highway: true
+  early_exit_entropy: -1
+
+model:
+  _target_: bert_squeeze.models.lt_deebert.LtDeeBert
+  training_config: ${train}
+  pretrained_model: "bert-base-cased"
+  num_labels: ${general.num_labels}
+
+data:
+  _target_: bert_squeeze.data.modules.transformer_module.TransformerDataModule
+  dataset_config:
+    is_local: false
+    path: emotion
+    split:
+    text_col: text
+    label_col: label
+    truncate_mode: head
+  tokenizer_name: ${model.pretrained_model}
+  max_length: 256

examples/configs/train_labse.yaml → bert_squeeze/assistants/configs/train_fastbert.yaml

@@ -1,53 +1,45 @@
-task:
-  name: train
-
 general:
+  debug: false
   do_train: true
   do_eval: false
-  debug: false
-  labels:
-    - sadness
-    - joy
-    - love
-    - anger
-    - fear
-    - surprise
+  get_mismatched: true
+  evaluate_during_training: true
+  labels: [ 0, 1 ]
   output_dir: outputs
   save_steps: 500
   validation_every_n_epoch: 1
-  evaluate_during_training: true
-  get_mismated: true
-  model_path: ../checkpoints/BER-511/
+
+callbacks:
+  - _target_: bert_squeeze.utils.callbacks.fastbert_logic.FastBertLogic
 
 train:
-  training_batch_size: 8
-  num_epochs: 10
-  dropout: 0.2
-  objective: lsl
-  smoothing: 0.1
-  optimizer: bertadam
-  weight_decay: 0.01
-  discriminative_learning: true
-  learning_rates: [ 2e-5 ]
+  adam_eps: 1e-8
+  accumulation_steps: 1
   auto_lr: false
+  discriminative_learning: true
+  dropout: 0.2
   layer_lr_decay: 0.95
+  learning_rates: [ 2e-5 ]
+  logging_steps: 50
   lr_scheduler: true
-  adam_eps: 1e-8
+  max_grad_norm: 1.0
+  num_epochs: 10
+  optimizer: bertadam
+  objective: lsl
+  smoothing: 0.1
   warmup_ratio: 0.06
   warmup_steps: true
-  max_grad_norm: 1.0
-  accumulation_steps: 1
-  eval_batch_size: 16
-  logging_steps: 50
+  weight_decay: 0.01
 
 model:
-  _target_: bert-squeeze.models.lt_labse.LtCustomLabse
+  _target_: bert_squeeze.models.lt_fastbert.LtFastBert
   training_config: ${train}
-  pretrained_model: "sentence-transformers/LaBSE"
-  num_labels: 6
+  pretrained_model: "bert-base-cased"
+  num_labels: 2
+  scorer_type: "fast"
 
 data:
-  _target_: bert-squeeze.data.modules.transformer_module.TransformerDataModule
+  _target_: bert_squeeze.data.modules.transformer_module.TransformerDataModule
   dataset_config:
     is_local: false
     path: emotion
@@ -56,16 +48,4 @@ data:
     label_col: label
     truncate_mode: head
   tokenizer_name: ${model.pretrained_model}
-  max_length: 256
-  train_batch_size: ${train.training_batch_size}
-  eval_batch_size: ${train.eval_batch_size}
-
-neptune:
-  user_name: julesbelveze
-  project: bert-tricks
-  tags: [ ]
-  logger:
-    _target_: neptune.new.integrations.pytorch_lightning.NeptuneLogger
-    project: ${neptune.user_name}/${neptune.project}
-    name: ${task.name}
-
+  max_length: 256

bert_squeeze/assistants/configs/train_lr.yaml

@@ -0,0 +1,42 @@
+general:
+  debug: false
+  do_train: true
+  do_eval: false
+  get_mismatched: true
+  evaluate_during_training: true
+  labels: [ 0,1 ]
+  num_labels: 2
+  output_dir: outputs
+  save_steps: 500
+  validation_every_n_epoch: 1
+
+train:
+  accumulation_steps: 1
+  auto_lr: false
+  discriminative_learning: false
+  eval_batch_size: 16
+  learning_rates: [ 2e-1 ]
+  logging_steps: 100
+  max_grad_norm: 1.0
+  num_epochs: 10
+  objective: ce
+  smoothing: 0.1
+  training_batch_size: 16
+
+model:
+  _target_: bert_squeeze.models.lr.BowLogisticRegression
+  training_config: ${train}
+  vocab_size: 5000
+  embed_dim: 256
+  num_labels: ${general.num_labels}
+  name: "bow_lr"
+
+data:
+  _target_: bert_squeeze.data.modules.lr_module.LrDataModule
+  dataset_config:
+    is_local: false
+    label_col: label
+    path:
+    split:
+    text_col: text
+  max_features: ${model.vocab_size}

bert_squeeze/assistants/configs/train_lstm.yaml

@@ -0,0 +1,40 @@
+general:
+  debug: false
+  do_train: true
+  do_eval: false
+  get_mismatched: true
+  evaluate_during_training: true
+  labels: [ 0,1 ]
+  num_labels: 2
+  output_dir: outputs
+  save_steps: 500
+  validation_every_n_epoch: 1
+
+train:
+  accumulation_steps: 1
+  auto_lr: false
+  dropout: 0.2
+  learning_rates: [ 2e-3 ]
+  logging_steps: 100
+  max_grad_norm: 1.0
+  num_epochs: 10
+  objective: ce
+  smoothing: 0.1
+
+model:
+  _target_: bert_squeeze.models.lstm.LtLSTM
+  hidden_dim: 128
+  name: lstm
+  num_labels: ${general.num_labels}
+  training_config: ${train}
+  vocab_size: 20000
+
+data:
+  _target_: bert_squeeze.data.modules.lstm_module.LSTMDataModule
+  dataset_config:
+    is_local: false
+    path: emotion
+    split:
+    text_col: text
+    label_col: label
+  max_features: ${model.vocab_size}

examples/configs/train_theseus_labse.yaml → bert_squeeze/assistants/configs/train_theseus_bert.yaml

@@ -1,57 +1,45 @@
-task:
-  name: train
-
 general:
+  debug: false
   do_train: true
   do_eval: false
-  debug: false
-  labels:
-    - sadness
-    - joy
-    - love
-    - anger
-    - fear
-    - surprise
+  get_mismatched: true
+  evaluate_during_training: true
+  labels: [ 0, 1 ]
   output_dir: outputs
   save_steps: 500
   validation_every_n_epoch: 1
-  evaluate_during_training: true
-  get_mismated: true
-  model_path: ../checkpoints/BER-511/
 
 train:
-  training_batch_size: 8
-  num_epochs: 10
-  dropout: 0.2
-  objective: lsl
-  smoothing: 0.1
-  optimizer: bertadam
-  weight_decay: 0.01
-  discriminative_learning: true
-  learning_rates: [ 2e-5 ]
+  adam_eps: 1e-8
+  accumulation_steps: 1
   auto_lr: false
+  discriminative_learning: true
+  dropout: 0.2
   layer_lr_decay: 0.95
+  learning_rates: [ 2e-5 ]
+  logging_steps: 50
   lr_scheduler: true
-  adam_eps: 1e-8
+  max_grad_norm: 1.0
+  num_epochs: 10
+  optimizer: bertadam
+  objective: lsl
+  smoothing: 0.1
   warmup_ratio: 0.06
   warmup_steps: true
-  max_grad_norm: 1.0
-  accumulation_steps: 1
-  eval_batch_size: 16
-  logging_steps: 50
+  weight_decay: 0.01
 
 model:
-  _target_: bert-squeeze.models.lt_theseus_bert.LtTheseusBert
+  _target_: bert_squeeze.models.lt_theseus_bert.LtTheseusBert
   training_config: ${train}
-  pretrained_model: "sentence-transformers/LaBSE"
-  num_labels: 6
+  pretrained_model: "bert-base-cased"
+  num_labels: 2
   replacement_scheduler:
     type: "linear"
     base_replacing_rate: 0.3
     coefficient: 0.0006
 
 data:
-  _target_: bert-squeeze.data.modules.transformer_module.TransformerDataModule
+  _target_: bert_squeeze.data.modules.transformer_module.TransformerDataModule
   dataset_config:
     is_local: false
     path: emotion
@@ -61,15 +49,3 @@ data:
     truncate_mode: head
   tokenizer_name: ${model.pretrained_model}
   max_length: 256
-  train_batch_size: ${train.training_batch_size}
-  eval_batch_size: ${train.eval_batch_size}
-
-neptune:
-  user_name: julesbelveze
-  project: bert-tricks
-  tags: [ ]
-  logger:
-    _target_: neptune.new.integrations.pytorch_lightning.NeptuneLogger
-    project: ${neptune.user_name}/${neptune.project}
-    name: ${task.name}
-