diff --git a/README.md b/README.md index 35c8ec33ea..0a41d05447 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Key features: 1. NLP: alpaca-en(gpt4), alpaca-zh(gpt4), finance-en, multi-alpaca-all, code-en, instinwild-en, instinwild-zh, cot-en, cot-zh, firefly-all-zh, poetry-zh, instruct-en, gpt4all-en 2. agent: [damo-agent-zh](https://modelscope.cn/datasets/damo/MSAgent-Bench/summary), damo-agent-mini-zh 3. multi-modal: coco-en -5. supported templates: chatml(qwen), baichuan, chatglm2, llama, openbuddy_llama, default +5. supported templates: chatml(qwen), baichuan, chatglm2, llama, openbuddy-llama, default # Installation diff --git a/README_CN.md b/README_CN.md index 3102114fbd..de739de3cd 100644 --- a/README_CN.md +++ b/README_CN.md @@ -36,7 +36,7 @@ SWIFT(Scalable lightWeight Infrastructure for Fine-Tuning)是一个可扩展 1. NLP: alpaca-en(gpt4), alpaca-zh(gpt4), finance-en, multi-alpaca-all, code-en, instinwild-en, instinwild-zh, cot-en, cot-zh, firefly-all-zh, poetry-zh, instruct-en, gpt4all-en 2. agent: [damo-agent-zh](https://modelscope.cn/datasets/damo/MSAgent-Bench/summary), damo-agent-mini-zh 3. multi-modal: coco-en -5. 支持的对话模板: chatml(qwen), baichuan, chatglm2, llama, openbuddy_llama, default +5. 支持的对话模板: chatml(qwen), baichuan, chatglm2, llama, openbuddy-llama, default # 安装 diff --git a/examples/pytorch/llm/README.md b/examples/pytorch/llm/README.md index e593d2f2cd..dd92594fea 100644 --- a/examples/pytorch/llm/README.md +++ b/examples/pytorch/llm/README.md @@ -16,13 +16,13 @@ ## Features 1. supported SFT methods: [lora](https://arxiv.org/abs/2106.09685), [qlora](https://arxiv.org/abs/2305.14314), full(full parameter fine-tuning) -2. supported models: qwen-7b, [qwen-7b-chat](https://github.com/QwenLM/Qwen-7B), qwen-vl, [qwen-vl-chat](https://github.com/QwenLM/Qwen-VL), baichuan-7b, baichuan-13b, baichuan-13b-chat, chatglm2-6b, chatglm2-6b-32k, llama2-7b, llama2-7b-chat, llama2-13b, llama2-13b-chat, llama2-70b, llama2-70b-chat, openbuddy-llama2-13b, openbuddy-llama-65b, polylm-13b +2. supported models: qwen-7b, [qwen-7b-chat](https://github.com/QwenLM/Qwen-7B), qwen-vl, [qwen-vl-chat](https://github.com/QwenLM/Qwen-VL), baichuan-7b, baichuan-13b, baichuan-13b-chat, chatglm2-6b, chatglm2-6b-32k, llama2-7b, llama2-7b-chat, llama2-13b, llama2-13b-chat, llama2-70b, llama2-70b-chat, openbuddy-llama2-13b, openbuddy-llama-65b, openbuddy-llama2-70b, polylm-13b 3. supported features: quantization, ddp, model parallelism(device map), gradient checkpointing, gradient accumulation, pushing to modelscope hub, custom datasets, multimodal and agent SFT, mutli-round chat, ... 4. supported datasets: 1. NLP: alpaca-en(gpt4), alpaca-zh(gpt4), finance-en, multi-alpaca-all, code-en, instinwild-en, instinwild-zh, cot-en, cot-zh, firefly-all-zh, poetry-zh, instruct-en, gpt4all-en 2. agent: [damo-agent-zh](https://modelscope.cn/datasets/damo/MSAgent-Bench/summary), damo-agent-mini-zh 3. multi-modal: coco-en -5. supported templates: chatml(qwen), baichuan, chatglm2, llama, openbuddy_llama, default +5. supported templates: chatml(qwen), baichuan, chatglm2, llama, openbuddy-llama, default ## Prepare the Environment Experimental environment: A10, 3090, A100, ... (V100 does not support bf16, quantization) diff --git a/examples/pytorch/llm/README_CN.md b/examples/pytorch/llm/README_CN.md index db14021b3d..91962f1882 100644 --- a/examples/pytorch/llm/README_CN.md +++ b/examples/pytorch/llm/README_CN.md @@ -17,13 +17,13 @@ ## 特性 1. 支持的SFT方法: [lora](https://arxiv.org/abs/2106.09685), [qlora](https://arxiv.org/abs/2305.14314), 全参数微调 -2. 支持的模型: qwen-7b, [qwen-7b-chat](https://github.com/QwenLM/Qwen-7B), qwen-vl, [qwen-vl-chat](https://github.com/QwenLM/Qwen-VL), baichuan-7b, baichuan-13b, baichuan-13b-chat, chatglm2-6b, chatglm2-6b-32k, llama2-7b, llama2-7b-chat, llama2-13b, llama2-13b-chat, llama2-70b, llama2-70b-chat, openbuddy-llama2-13b, openbuddy-llama-65b, polylm-13b +2. 支持的模型: qwen-7b, [qwen-7b-chat](https://github.com/QwenLM/Qwen-7B), qwen-vl, [qwen-vl-chat](https://github.com/QwenLM/Qwen-VL), baichuan-7b, baichuan-13b, baichuan-13b-chat, chatglm2-6b, chatglm2-6b-32k, llama2-7b, llama2-7b-chat, llama2-13b, llama2-13b-chat, llama2-70b, llama2-70b-chat, openbuddy-llama2-13b, openbuddy-llama-65b, openbuddy-llama2-70b, polylm-13b 3. 支持的特性: 模型量化, DDP, 模型并行(device_map), gradient checkpointing, 梯度累加, 支持推送ModelScope Hub, 自定义数据集, 多模态和Agent SFT, 多轮对话, ... 4. 支持的数据集: 1. NLP: alpaca-en(gpt4), alpaca-zh(gpt4), finance-en, multi-alpaca-all, code-en, instinwild-en, instinwild-zh, cot-en, cot-zh, firefly-all-zh, poetry-zh, instruct-en, gpt4all-en 2. agent: [damo-agent-zh](https://modelscope.cn/datasets/damo/MSAgent-Bench/summary), damo-agent-mini-zh 3. multi-modal: coco-en -5. 支持的对话模板: chatml(qwen), baichuan, chatglm2, llama, openbuddy_llama, default +5. 支持的对话模板: chatml(qwen), baichuan, chatglm2, llama, openbuddy-llama, default ## 准备实验环境 实验环境: A10, 3090, A100均可. (V100不支持bf16, 量化) diff --git a/examples/pytorch/llm/scripts/openbuddy-llama-65b/qlora/infer.sh b/examples/pytorch/llm/scripts/openbuddy-llama2-70b/qlora/infer.sh similarity index 53% rename from examples/pytorch/llm/scripts/openbuddy-llama-65b/qlora/infer.sh rename to examples/pytorch/llm/scripts/openbuddy-llama2-70b/qlora/infer.sh index 64dabe1807..110b2850cc 100644 --- a/examples/pytorch/llm/scripts/openbuddy-llama-65b/qlora/infer.sh +++ b/examples/pytorch/llm/scripts/openbuddy-llama2-70b/qlora/infer.sh @@ -1,11 +1,14 @@ -# 38G +# 40G CUDA_VISIBLE_DEVICES=0,1 \ python src/llm_infer.py \ - --model_type openbuddy-llama-65b \ + --model_type openbuddy-llama2-70b \ --sft_type lora \ - --ckpt_dir "runs/openbuddy-llama-65b/vx_xxx/checkpoint-xxx" \ + --template_type openbuddy-llama \ + --dtype bf16 \ + --ckpt_dir "runs/openbuddy-llama2-70b/vx_xxx/checkpoint-xxx" \ --eval_human true \ --quantization_bit 4 \ + --bnb_4bit_comp_dtype bf16 \ --max_new_tokens 1024 \ --temperature 0.9 \ --top_k 50 \ diff --git a/examples/pytorch/llm/scripts/openbuddy-llama-65b/qlora/sft.sh b/examples/pytorch/llm/scripts/openbuddy-llama2-70b/qlora/sft.sh similarity index 55% rename from examples/pytorch/llm/scripts/openbuddy-llama-65b/qlora/sft.sh rename to examples/pytorch/llm/scripts/openbuddy-llama2-70b/qlora/sft.sh index ea2feb670a..0b9acb6682 100644 --- a/examples/pytorch/llm/scripts/openbuddy-llama-65b/qlora/sft.sh +++ b/examples/pytorch/llm/scripts/openbuddy-llama2-70b/qlora/sft.sh @@ -1,21 +1,32 @@ -# 40G +# 42G VRAM CUDA_VISIBLE_DEVICES=0,1 \ python src/llm_sft.py \ - --model_type openbuddy-llama-65b \ + --model_type openbuddy-llama2-70b \ --sft_type lora \ + --template_type openbuddy-llama \ + --dtype bf16 \ --output_dir runs \ --dataset alpaca-en,alpaca-zh \ --dataset_sample 20000 \ --num_train_epochs 1 \ --max_length 1024 \ --quantization_bit 4 \ + --bnb_4bit_comp_dtype bf16 \ --lora_rank 8 \ --lora_alpha 32 \ --lora_dropout_p 0.1 \ + --gradient_checkpointing true \ --batch_size 1 \ + --weight_decay 0. \ --learning_rate 1e-4 \ --gradient_accumulation_steps 16 \ + --max_grad_norm 0.5 \ + --warmup_ratio 0.03 \ --eval_steps 50 \ --save_steps 50 \ --save_total_limit 2 \ --logging_steps 10 \ + --push_to_hub false \ + --hub_model_id openbuddy-llama2-70b-qlora \ + --hub_private_repo true \ + --hub_token 'your-sdk-token' \ diff --git a/examples/pytorch/llm/src/utils/model.py b/examples/pytorch/llm/src/utils/model.py index 2ab075c35d..281e7651cd 100644 --- a/examples/pytorch/llm/src/utils/model.py +++ b/examples/pytorch/llm/src/utils/model.py @@ -278,13 +278,19 @@ class LoRATM(NamedTuple): 'openbuddy-llama2-13b': { 'model_id': 'OpenBuddy/openbuddy-llama2-13b-v8.1-fp16', 'revision': 'v1.0.0', - 'template': 'openbuddy_llama', + 'template': 'openbuddy-llama', 'lora_TM': LoRATM.llama2, }, 'openbuddy-llama-65b': { 'model_id': 'OpenBuddy/openbuddy-llama-65b-v8-bf16', 'revision': 'v1.0.0', - 'template': 'openbuddy_llama', + 'template': 'openbuddy-llama', + 'lora_TM': LoRATM.llama2, + }, + 'openbuddy-llama2-70b': { + 'model_id': 'OpenBuddy/openbuddy-llama2-70b-v10.1-bf16', + 'revision': 'v1.0.0', + 'template': 'openbuddy-llama', 'lora_TM': LoRATM.llama2, }, 'polylm-13b': { diff --git a/examples/pytorch/llm/src/utils/preprocess.py b/examples/pytorch/llm/src/utils/preprocess.py index 8045f406d5..afdb01bb47 100644 --- a/examples/pytorch/llm/src/utils/preprocess.py +++ b/examples/pytorch/llm/src/utils/preprocess.py @@ -42,7 +42,7 @@ 'chat_sep': [' ', ['eos_token_id', 'bos_token_id'], '[INST] '], 'suffix': [['eos_token_id']], }, - 'openbuddy_llama': { + 'openbuddy-llama': { 'prefix': ['{{system}}\n\n'], 'prompt': ['User: {{query}}\nAssistant: '], 'chat_sep': ['\n'], diff --git a/swift/trainers/mixin.py b/swift/trainers/mixin.py index af133fe8db..2b785d99d5 100644 --- a/swift/trainers/mixin.py +++ b/swift/trainers/mixin.py @@ -26,7 +26,7 @@ from swift.utils.constants import Invoke from swift.utils.logger import get_logger from .utils import (can_return_loss, find_labels, get_function, - is_instance_of_ms_Model) + is_instance_of_ms_model) logger = get_logger() @@ -229,7 +229,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None): output_dir = output_dir if output_dir is not None else self.args.output_dir os.makedirs(output_dir, exist_ok=True) logger.info(f'Saving model checkpoint to {output_dir}') - if is_instance_of_ms_Model(self.model): + if is_instance_of_ms_model(self.model): model_dir = getattr(self.model, 'model_dir', None) if model_dir is not None: src_path = os.path.join(model_dir, 'configuration.json') @@ -263,7 +263,7 @@ def _save(self, output_dir: Optional[str] = None, state_dict=None): else: torch.save(state_dict, os.path.join(output_dir, 'pytorch_model.bin')) - elif is_instance_of_ms_Model(self.model): + elif is_instance_of_ms_model(self.model): PreTrainedModel.save_pretrained( self.model, output_dir, diff --git a/swift/trainers/utils.py b/swift/trainers/utils.py index a94214a8e6..8815566169 100644 --- a/swift/trainers/utils.py +++ b/swift/trainers/utils.py @@ -37,7 +37,7 @@ def get_function( return method_or_function -def is_instance_of_ms_Model(model: Module) -> bool: +def is_instance_of_ms_model(model: Module) -> bool: """avoid import modelscope: circular dependency problem""" for m_cls in model.__class__.__mro__: cls_name = m_cls.__name__ diff --git a/swift/utils/__init__.py b/swift/utils/__init__.py index f30ac2801f..35a8e6bfc2 100644 --- a/swift/utils/__init__.py +++ b/swift/utils/__init__.py @@ -2,6 +2,6 @@ from .logger import get_logger from .torch_utils import (add_version_to_work_dir, broadcast_string, - get_dist_setting, get_seed, is_dist, is_master, - is_on_same_device, parse_args, print_model_info, - seed_everything, show_layers) + get_dist_setting, get_seed, is_dist, is_local_master, + is_master, is_on_same_device, parse_args, + print_model_info, seed_everything, show_layers) diff --git a/swift/utils/torch_utils.py b/swift/utils/torch_utils.py index 4fca0a28d5..d993c7318d 100644 --- a/swift/utils/torch_utils.py +++ b/swift/utils/torch_utils.py @@ -125,6 +125,11 @@ def get_dist_setting() -> Tuple[int, int, int, int]: return rank, local_rank, world_size, local_world_size +def is_local_master(): + local_rank = get_dist_setting()[1] + return local_rank in {-1, 0} + + def is_dist(): """Determine if the training is distributed""" rank, local_rank, _, _ = get_dist_setting()