Revert "PL: --adafactor option (huggingface#6776)"

This reverts commit 1179f87.

examples/lightning_base.py

@@ -22,7 +22,6 @@
     PreTrainedTokenizer,
 )
 from transformers.optimization import (
-    Adafactor,
     get_cosine_schedule_with_warmup,
     get_cosine_with_hard_restarts_schedule_with_warmup,
     get_linear_schedule_with_warmup,
@@ -138,15 +137,7 @@ def configure_optimizers(self):
                 "weight_decay": 0.0,
             },
         ]
-        if self.hparams.adafactor:
-            optimizer = Adafactor(
-                optimizer_grouped_parameters, lr=self.hparams.learning_rate, scale_parameter=False, relative_step=False
-            )
-
-        else:
-            optimizer = AdamW(
-                optimizer_grouped_parameters, lr=self.hparams.learning_rate, eps=self.hparams.adam_epsilon
-            )
+        optimizer = AdamW(optimizer_grouped_parameters, lr=self.hparams.learning_rate, eps=self.hparams.adam_epsilon)
         self.opt = optimizer
 
         scheduler = self.get_lr_scheduler()
@@ -260,7 +251,6 @@ def add_model_specific_args(parser, root_dir):
         parser.add_argument("--num_train_epochs", dest="max_epochs", default=3, type=int)
         parser.add_argument("--train_batch_size", default=32, type=int)
         parser.add_argument("--eval_batch_size", default=32, type=int)
-        parser.add_argument("--adafactor", action="store_true")
 
 
 class LoggingCallback(pl.Callback):

examples/seq2seq/test_seq2seq_examples.py

@@ -30,7 +30,6 @@
 CUDA_AVAILABLE = torch.cuda.is_available()
 CHEAP_ARGS = {
     "label_smoothing": 0.2,
-    "adafactor": True,
     "early_stopping_patience": 2,
     "logger_name": "default",
     "length_penalty": 0.5,