Merge pull request #799 from FeixLiu/sharding_overlap

support sharding overlap
PaddlePaddle · Sep 29, 2022 · 23e0136 · 23e0136
2 parents b34dc2b + bacb2f6
commit 23e0136
Show file tree

Hide file tree

Showing 12 changed files with 19 additions and 0 deletions.
diff --git a/ppfleetx/configs/nlp/gpt/finetune_gpt_345M_single_card_glue.yaml b/ppfleetx/configs/nlp/gpt/finetune_gpt_345M_single_card_glue.yaml
@@ -61,6 +61,7 @@ Distributed:
     sharding_degree: 1
     sharding_stage: 1
     sharding_offload: False
+    comm_overlap: False
 
 Optimizer:
   name: FusedAdamW

diff --git a/ppfleetx/configs/nlp/gpt/generation_gpt_345M_dp8.yaml b/ppfleetx/configs/nlp/gpt/generation_gpt_345M_dp8.yaml
@@ -20,3 +20,4 @@ Distributed:
     sharding_degree: 1
     sharding_stage: 1
     sharding_offload: False
+    comm_overlap: False
diff --git a/ppfleetx/configs/nlp/gpt/inference_gpt_345M_dp8.yaml b/ppfleetx/configs/nlp/gpt/inference_gpt_345M_dp8.yaml
@@ -14,6 +14,7 @@ Distributed:
     sharding_degree: 1
     sharding_stage: 1
     sharding_offload: False
+    comm_overlap: False
 
 
 Data:

diff --git a/ppfleetx/configs/nlp/gpt/inference_gpt_345M_single_card.yaml b/ppfleetx/configs/nlp/gpt/inference_gpt_345M_single_card.yaml
@@ -14,6 +14,7 @@ Distributed:
     sharding_degree: 1
     sharding_stage: 1
     sharding_offload: False
+    comm_overlap: False
 
 
 Data:

diff --git a/ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B_dp8.yaml b/ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B_dp8.yaml
@@ -30,3 +30,4 @@ Distributed:
     sharding_degree: 1
     sharding_stage: 1
     sharding_offload: False
+    comm_overlap: False
diff --git a/ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B_single_card.yaml b/ppfleetx/configs/nlp/gpt/pretrain_gpt_1.3B_single_card.yaml
@@ -30,3 +30,4 @@ Distributed:
     sharding_degree: 1
     sharding_stage: 1
     sharding_offload: False
+    comm_overlap: False
diff --git a/ppfleetx/configs/nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml b/ppfleetx/configs/nlp/gpt/pretrain_gpt_175B_mp8_pp16.yaml
@@ -31,3 +31,4 @@ Distributed:
     sharding_degree: 1
     sharding_stage: 1
     sharding_offload: False
+    comm_overlap: False
diff --git a/ppfleetx/configs/nlp/gpt/pretrain_gpt_345M_mp8_qat.yaml b/ppfleetx/configs/nlp/gpt/pretrain_gpt_345M_mp8_qat.yaml
@@ -30,6 +30,7 @@ Distributed:
     sharding_degree: 1
     sharding_stage: 1
     sharding_offload: False
+    comm_overlap: False
 
 
 Quantization:

diff --git a/ppfleetx/configs/nlp/gpt/pretrain_gpt_345M_single_card.yaml b/ppfleetx/configs/nlp/gpt/pretrain_gpt_345M_single_card.yaml
@@ -30,3 +30,4 @@ Distributed:
     sharding_degree: 1
     sharding_stage: 1
     sharding_offload: False
+    comm_overlap: False
diff --git a/ppfleetx/configs/nlp/gpt/pretrain_gpt_6.7B_sharding16.yaml b/ppfleetx/configs/nlp/gpt/pretrain_gpt_6.7B_sharding16.yaml
@@ -30,3 +30,4 @@ Distributed:
     sharding_degree: 16
     sharding_stage: 2
     sharding_offload: False
+    comm_overlap: True
diff --git a/ppfleetx/core/engine/eager_engine.py b/ppfleetx/core/engine/eager_engine.py
@@ -153,6 +153,11 @@ def configure_optimizers(self):
             'sharding_degree']
         self._sharding_offload = self._dist_configs['sharding'][
             'sharding_offload']
+        self._comm_overlap = self._dist_configs['sharding']['comm_overlap']
+        if self._sharding_degree > 1 and self._comm_overlap:
+            if self._sharding_stage == 3 or self._sharding_offload:
+                self._comm_overlap = False
+                logger.warning("comm overlap only valid for sharding stage 2 without offload")
         self._use_recompute = configs['Model']['use_recompute']
 
         if self._use_pure_fp16:
@@ -245,6 +250,8 @@ def _wrap_sharding_2_3(self):
             scaler=self._scaler,
             group=self._sharding_group,
             offload=self._sharding_offload)
+        if self._comm_overlap:
+            self._module.model._set_comm_overlap(self._comm_overlap)
 
     def _wrap_3D_parallel(self):
         self._module.model = fleet.distributed_model(self._module.model)

diff --git a/projects/gpt/docs/hybrid_parallel.md b/projects/gpt/docs/hybrid_parallel.md
@@ -26,6 +26,7 @@
       sharding_degree: 1
       sharding_stage: 1
       sharding_offload: False
+      comm_overlap: False
 ```
 
 其中参数说明：
@@ -38,6 +39,7 @@
 | sharding_degree  | 分组切分并行维度                             |
 | sharding_stage   | 切分策略；1表示仅切分优化器状态，2表示再切分梯度，3表示再切分前向参数 |
 | sharding_offload | CPU offload策略                        |
+|comm_overlap| 是否在sharding stage 2的模式下进行通讯与计算overlap，该策略暂时不支持sharding_offload|
 
 ## 运行方式
 本目录中按照345M、1.3B、6.7B和175B规模大小，给出32G V100环境下GPT模型混合并行训练的策略配置如下：