add support for adamw schedulefree

[winglian]

implements meta's https://github.com/facebookresearch/schedule_free for adamw

https://twitter.com/aaron_defazio/status/1776320004465582331

optimizer: schedule_free_adamw
lr_scheduler: constant

[winglian]

upstreaming this @ huggingface/transformers#30079

[seungduk-yanolja]

I got the following error:

Traceback (most recent call last):
  File "/home/seungduk/miniconda3/envs/axo/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/home/seungduk/miniconda3/envs/axo/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/data/shared/apps/axolotl/src/axolotl/cli/train.py", line 59, in <module>
    fire.Fire(do_cli)
  File "/home/seungduk/miniconda3/envs/axo/lib/python3.10/site-packages/fire/core.py", line 141, in Fire
    component_trace = _Fire(component, args, parsed_flag_args, context, name)
  File "/home/seungduk/miniconda3/envs/axo/lib/python3.10/site-packages/fire/core.py", line 475, in _Fire
    component, remaining_args = _CallAndUpdateTrace(
  File "/home/seungduk/miniconda3/envs/axo/lib/python3.10/site-packages/fire/core.py", line 691, in _CallAndUpdateTrace
    component = fn(*varargs, **kwargs)
  File "/data/shared/apps/axolotl/src/axolotl/cli/train.py", line 35, in do_cli
    return do_train(parsed_cfg, parsed_cli_args)
  File "/data/shared/apps/axolotl/src/axolotl/cli/train.py", line 55, in do_train
    return train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
  File "/data/shared/apps/axolotl/src/axolotl/train.py", line 104, in train
    trainer = setup_trainer(
  File "/data/shared/apps/axolotl/src/axolotl/utils/trainer.py", line 351, in setup_trainer
    return trainer_builder.build(total_num_steps)
  File "/data/shared/apps/axolotl/src/axolotl/core/trainer_builder.py", line 1377, in build
    trainer = trainer_cls(
  File "/data/shared/apps/axolotl/src/axolotl/core/trainer_builder.py", line 239, in __init__
    super().__init__(*_args, **kwargs)
  File "/home/seungduk/miniconda3/envs/axo/lib/python3.10/site-packages/transformers/trainer.py", line 537, in __init__
    raise RuntimeError(
RuntimeError: Passing `optimizers` is not allowed if Deepspeed or PyTorch FSDP is enabled. You should subclass `Trainer` and override the `create_optimizer_and_scheduler` method.