add accelerate user guides

Signed-off-by: woshiyyya <xiaoyunxuan1998@gmail.com>

doc/BUILD

@@ -225,8 +225,6 @@ py_test_run_all_subdirectory(
     include = ["source/train/doc_code/*.py"],
     exclude = [
         "source/train/doc_code/hf_trainer.py",  # Too large
-        "source/train/doc_code/accelerate_torch_trainer.py", # GPU test
-        "source/train/doc_code/deepspeed_torch_trainer.py",  # GPU test
     ],
     extra_srcs = [],
     tags = ["exclusive", "team:ml"],
@@ -272,17 +270,6 @@ py_test(
     args = ["--path", "doc/source/train/examples/pytorch/pytorch_resnet_finetune.ipynb"]
 )
 
-py_test_run_all_subdirectory(
-    size = "large",
-    include = [
-        "source/train/doc_code/accelerate_torch_trainer.py", # GPU test
-        "source/train/doc_code/deepspeed_torch_trainer.py",  # GPU test
-    ],
-    exclude = [],
-    extra_srcs = [],
-    tags = ["exclusive", "team:ml", "gpu"],
-)
-
 # --------------------------------------------------------------------
 # Test all doc/external code
 # --------------------------------------------------------------------

doc/source/train/deepspeed.rst

@@ -0,0 +1,12 @@
+.. _train-deepspeed:
+
+Training with DeepSpeed
+=======================
+
+
+.. dropdown:: Code example
+
+    .. literalinclude:: ./doc_code/deepspeed_torch_trainer.py
+        :language: python
+        :start-after: __deepspeed_torch_basic_example_start__
+        :end-before: __deepspeed_torch_basic_example_end__

doc/source/train/doc_code/accelerate_trainer.py

@@ -52,7 +52,7 @@ def train_loop_per_worker():
             print(f"epoch: {epoch}, loss: {loss.item()}")
 
         train.report(
-            {},
+            metrics={"epoch": epoch, "loss": loss.item()},
             checkpoint=Checkpoint.from_dict(
                 dict(epoch=epoch, model=accelerator.unwrap_model(model).state_dict())
             ),

doc/source/train/examples/accelerate/accelerate_example.rst

@@ -0,0 +1,8 @@
+:orphan:
+
+.. _accelerate_example:
+
+Hugging Face Accelerate Distributed Training Example with Ray Train
+===================================================================
+
+.. literalinclude:: /../../python/ray/train/examples/accelerate/accelerate_torch_trainer.py

doc/source/train/examples/deepspeed/deepspeed_example.rst

@@ -0,0 +1,8 @@
+:orphan:
+
+.. _deepspeed_example:
+
+DeepSpeed ZeRO-3 Distributed Training Example with Ray Train
+============================================================
+
+.. literalinclude:: /../../python/ray/train/examples/deepspeed/deepspeed_torch_trainer.py