[New features] testing example of model zoo (#4398)

* update model-zoo test

* update config

* update model-zoo test

* update config for gpt

* add from_hf_hubarg

* update model-zoo gpt

* add export & infer test

* use with argv context

* update gpt testing

* update gpt configuration

* skip gpt-ci

* fix lint

* Update tests/model_zoo/test_gpt.py

Co-authored-by: Zhong Hui <zhonghui.net@gmail.com>

* update ci-case & config variable name

---------

Co-authored-by: Zhong Hui <zhonghui.net@gmail.com>

model_zoo/gpt/run_generation.py

@@ -40,6 +40,7 @@ def parse_args():
         type=str,
         help="The path or shortcut name of the pre-trained model.",
     )
+    parser.add_argument("--from_hf_hub", type=bool, default=False, help="Whether load model from hf hub")
     parser.add_argument(
         "--decode_strategy", type=str, default="greedy_search", help="The decode strategy in generation."
     )
@@ -112,8 +113,8 @@ def main(args, input_text):
             )
         )
 
-    model = model_class.from_pretrained(args.model_name_or_path)
-    tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path)
+    model = model_class.from_pretrained(args.model_name_or_path, from_hf_hub=args.from_hf_hub)
+    tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path, from_hf_hub=args.from_hf_hub)
     model.eval()
 
     args.max_dec_len = adjust_length_to_model(args.max_dec_len, model.max_position_embeddings)

model_zoo/gpt/run_glue.py

@@ -18,7 +18,6 @@
 
 import numpy as np
 import paddle
-from args import parse_config_file
 from paddle.io import DataLoader
 from paddle.metric import Accuracy
 
@@ -224,11 +223,4 @@ def do_train():
 
 
 if __name__ == "__main__":
-    # support: python run_glue.py --config=./configs/default.yaml
-    config_file = parse_config_file()
-    if config_file is not None:
-        from args import init_argv
-
-        init_argv("glue", config_file)
-
     do_train()

model_zoo/gpt/run_msra_ner.py

@@ -17,7 +17,6 @@
 from functools import partial
 
 import paddle
-from args import init_argv, parse_config_file
 
 from paddlenlp.data import DataCollatorForTokenClassification
 from paddlenlp.datasets import load_dataset
@@ -154,7 +153,4 @@ def do_train():
 
 
 if __name__ == "__main__":
-    config_file = parse_config_file()
-    if config_file is not None:
-        init_argv("msra_ner", config_file)
     do_train()

model_zoo/gpt/run_pretrain.py

@@ -61,6 +61,7 @@ def eval_freq(self):
 
 @dataclass
 class ModelArguments:
+    model_type: str = field(default="gpt", metadata={"help": "the type of model"})
     model_name_or_path: str = field(default="gpt2-en", metadata={"help": ""})
     max_seq_len: int = field(default=128, metadata={"help": "max sequence length"})
     to_static: bool = field(default=False, metadata={"help": "whether use static pretraining mode."})
@@ -369,9 +370,10 @@ def do_train():
     # Now, we only support data parallel in dygraph mode for now.
     topo = Topology(device_rank=worker_index, world_size=worker_num, dp_degree=worker_num)
 
-    tokenizer = GPTTokenizer.from_pretrained(model_args.model_name_or_path)
-    pretrained_models_list = list(GPTForPretraining.pretrained_init_configuration.keys())
-    model = GPTForPretraining.from_pretrained(model_args.model_name_or_path)
+    model_class, tokenizer_class = MODEL_CLASSES[model_args.model_type]
+    tokenizer = tokenizer_class.from_pretrained(model_args.model_name_or_path)
+    pretrained_models_list = list(model_class.pretrained_init_configuration.keys())
+    model = model_class.from_pretrained(model_args.model_name_or_path)
 
     # Create the critrion for the gpt model
     criterion = GPTPretrainingCriterion()

scripts/regression/ci_case.sh

@@ -279,53 +279,8 @@ print_info $? gpt_deploy_C_FT
 }
 # 8 gpt
 gpt(){
-if [ ! -f 'test.py' ];then
-    echo '模型测试文件不存在！'
-    # data process
-    cd ${nlp_dir}/model_zoo/ernie-1.0/data_tools
-    sed -i "s/python3/python/g" Makefile
-    sed -i "s/python-config/python3.7m-config/g" Makefile
-    cd ${nlp_dir}/model_zoo/gpt/
-    # pretrain
-    python -m paddle.distributed.launch run_pretrain.py \
-    --model_name_or_path "__internal_testing__/gpt" \
-    --input_dir "./pre_data" \
-    --output_dir "output" \
-    --weight_decay 0.01 \
-    --max_steps 2 \
-    --save_steps 2 \
-    --device gpu \
-    --warmup_steps 320000 \
-    --warmup_ratio 0.01 \
-    --micro_batch_size 8 \
-    --eval_steps 100 \
-    --overwrite_output_dir true \
-    --dataloader_drop_last true \
-    --do_train true \
-    --do_predict true >${log_path}/gpt_pretrain >>${log_path}/gpt_pretrain 2>&1
-    print_info $? gpt_pretrain
-    # export model
-    python export_model.py --model_type=gpt \
-        --model_path=gpt2-medium-en \
-        --output_path=./infer_model/model >${log_path}/gpt_export >>${log_path}/gpt_export 2>&1
-    print_info $? gpt_export
-    # inference
-    python deploy/python/inference.py \
-        --model_type gpt \
-        --model_path ./infer_model/model >${log_path}/gpt_p_depoly >>${log_path}/gpt_p_depoly 2>&1
-    print_info $? gpt_p_depoly
-    # test acc
-    # cd ${nlp_dir}/tests/examples/gpt/
-    # time (python -m unittest test_accuracy.py >${log_path}/gpt_test_acc) >>${log_path}/gpt_test_acc 2>&1
-    # print_info $? gpt_test_acc
-else 
-    pytest ${nlp_dir}/model_zoo/gpt/ >${log_path}/gpt >>${log_path}/gpt 2>&1
-    print_info $? gpt
-fi
-fast_gpt
-cd ${nlp_dir}/fast_generation/samples
-python gpt_sample.py >${log_path}/fast_generation_gpt >>${log_path}/fast_generation_gpt 2>&1
-print_info $? fast_generation_gpt
+# TODO(wj-Mcat): need remove the gpt related code scripts in paddle-ci
+echo 'skip gpt testing in paddle-ci, for details you can see: https://github.com/PaddlePaddle/PaddleNLP/pull/4398'
 }
 # 9 ernie-1.0
 ernie-1.0 (){

tests/fixtures/model_zoo/gpt.yaml

@@ -0,0 +1,95 @@
+pretrain:
+  slow:
+    model_type: gpt
+    model_name_or_path: __internal_testing__/gpt
+    input_dir: ./data
+    output_dir: ./output_dir/pretrain
+    weight_decay: 0.01
+    max_steps: 2
+    save_steps: 10
+    warmup_steps: 10
+    warmup_ratio: 0.01
+    per_device_train_batch_size: 4
+    device: cpu
+    eval_steps: 10
+    do_train: true
+    do_predict: true
+
+  default:
+    model_type: gpt
+    model_name_or_path: gpt2-en
+    warmup_steps: 320000
+    warmup_ratio: 0.01
+    per_device_train_batch_size: 4
+    device: cpu
+    eval_steps: 100
+    do_train: true
+    do_predict: true
+
+
+msra_ner:
+  slow:
+    model_name_or_path: __internal_testing__/gpt_cn
+    max_seq_length: 32
+    per_device_eval_batch_size: 4
+    learning_rate: 2e-5
+    num_train_epochs: 3
+    logging_steps: 25
+    save_steps: 250
+    output_dir: ./output_dir/msra_ner/
+    device: cpu
+    do_train: true
+    max_steps: 10
+
+  default:
+    model_name_or_path: gpt-cpm-small-cn-distill
+    max_seq_length: 128
+    per_device_eval_batch_size: 32
+    learning_rate: 2e-5
+    num_train_epochs: 3
+    logging_steps: 25
+    save_steps: 250
+    output_dir: ./tmp/msra_ner/
+    device: gpu 
+    do_train: true
+
+glue:
+  slow:
+    model_name_or_path: __internal_testing__/gpt
+    task_name: SST-2
+    max_seq_length: 128
+    per_device_train_batch_size: 32  
+    learning_rate: 2e-5
+    num_train_epochs: 3
+    logging_steps: 1
+    save_steps: 500
+    output_dir: ./output_dir/glue
+    max_steps: 10
+    eval_steps: 1
+    device: cpu
+    do_train: true
+
+  default:
+    model_name_or_path: gpt2-medium-en
+    task_name: SST-2
+    max_seq_length: 128
+    per_device_train_batch_size: 32  
+    learning_rate: 2e-5
+    num_train_epochs: 3
+    logging_steps: 1
+    save_steps: 500
+    output_dir: ./output_dir/glue
+    eval_steps: 1
+    device: gpu
+    do_train: true
+
+
+generation:
+  slow:
+    model_type: gpt2
+    model_name_or_path: hf-internal-testing/tiny-random-GPT2Model
+    from_hf_hub: true
+
+  default:
+    model_type: gpt2-cn
+    model_name_or_path: gpt-cpm-small-cn-distill

tests/model_zoo/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.