训练时报错

[Brain2nd]

我自定义修改了如下脚本：

# Copyright (c) OpenMMLab. All rights reserved.
import torch
from datasets import load_dataset
from mmengine.dataset import DefaultSampler
from mmengine.hooks import (CheckpointHook, DistSamplerSeedHook, IterTimerHook,
                            LoggerHook, ParamSchedulerHook)
from mmengine.optim import AmpOptimWrapper, CosineAnnealingLR, LinearLR
from peft import LoraConfig
from torch.optim import AdamW
from transformers import (AutoModelForCausalLM, AutoTokenizer,
                          BitsAndBytesConfig)

from xtuner.dataset import process_hf_dataset
from xtuner.dataset.collate_fns import default_collate_fn
from xtuner.dataset.map_fns import alpaca_zh_map_fn, template_map_fn_factory
from xtuner.engine import DatasetInfoHook, EvaluateChatHook
from xtuner.model import SupervisedFinetune
from xtuner.utils import PROMPT_TEMPLATE, SYSTEM_TEMPLATE

#######################################################################
#                          PART 1  Settings                           #
#######################################################################
# Model
pretrained_model_name_or_path = '/home/tangshi/TangShi/Pku政务大模型/Models/Qwen-1_8B-Chat'

# Data
alpaca_zh_path = '/home/tangshi/TangShi/Pku政务大模型/Trainer/Tools/LsDtata/Data/pku'
prompt_template = PROMPT_TEMPLATE.qwen_chat
max_length = 2048
pack_to_max_length = True

# Scheduler & Optimizer
batch_size = 1  # per_device
accumulative_counts = 16
dataloader_num_workers = 0
max_epochs = 3
optim_type = AdamW
lr = 2e-4
betas = (0.9, 0.999)
weight_decay = 0
max_norm = 1  # grad clip
warmup_ratio = 0.03

# Evaluate the generation performance during the training
evaluation_freq = 500
SYSTEM = "你的任务是重庆市政务文书写作、政务问答 \n 你生成的问题必须包含：1、留言标题，2、留言摘要。你生成的答复内容部分必须有法律依据，且表明已审查，参照你固有的知识或者我给出的法律文献，在引用法律文件时使用《》包裹其名称。\n"
evaluation_inputs = [
    '留言标题：合川区一超市怀疑出售过期食品\n留言摘要：市民在合川区一超市发现了怀疑是过期的食品,担心会对消费者的健康造成威胁。', '留言标题：渝北区小区内共享单车难以管理\n留言摘要：渝北区某小区内共享单车聚集,严重影响居民出行,请求相关部门加强管理。'
]

#######################################################################
#                      PART 2  Model & Tokenizer                      #
#######################################################################
tokenizer = dict(
    type=AutoTokenizer.from_pretrained,
    pretrained_model_name_or_path=pretrained_model_name_or_path,
    trust_remote_code=True,
    padding_side='right',
    eos_token='<|im_end|>')

# model = dict(
#     type=SupervisedFinetune,
#     llm=dict(
#         type=AutoModelForCausalLM.from_pretrained,
#         pretrained_model_name_or_path=pretrained_model_name_or_path,
#         trust_remote_code=True,
#         torch_dtype=torch.float16,
        # quantization_config=dict(
        #     type=BitsAndBytesConfig,
        #     load_in_4bit=True,
        #     load_in_8bit=False,
        #     llm_int8_threshold=6.0,
        #     llm_int8_has_fp16_weight=False,
        #     bnb_4bit_compute_dtype=torch.float16,
        #     bnb_4bit_use_double_quant=True,
        #     bnb_4bit_quant_type='nf4')
        # ),
    # lora=dict(
    #     type=LoraConfig,
    #     r=64,
    #     lora_alpha=16,
    #     lora_dropout=0.1,
    #     bias='none',
    #     task_type='CAUSAL_LM')
    # )
model = dict(
    type=SupervisedFinetune,
    llm=dict(
        type=AutoModelForCausalLM.from_pretrained,
        pretrained_model_name_or_path=pretrained_model_name_or_path,
        trust_remote_code=True,
        torch_dtype=torch.float16,
    ),
)
#######################################################################
#                      PART 3  Dataset & Dataloader                   #
#######################################################################
alpaca_zh = dict(
    type=process_hf_dataset,
    dataset=dict(type=load_dataset, path=alpaca_zh_path),
    tokenizer=tokenizer,
    max_length=max_length,
    dataset_map_fn=alpaca_zh_map_fn,
    template_map_fn=dict(
        type=template_map_fn_factory, template=prompt_template),
    remove_unused_columns=True,
    shuffle_before_pack=True,
    pack_to_max_length=pack_to_max_length)

train_dataloader = dict(
    batch_size=batch_size,
    num_workers=dataloader_num_workers,
    dataset=alpaca_zh,
    sampler=dict(type=DefaultSampler, shuffle=True),
    collate_fn=dict(type=default_collate_fn))

#######################################################################
#                    PART 4  Scheduler & Optimizer                    #
#######################################################################
# optimizer
optim_wrapper = dict(
    type=AmpOptimWrapper,
    optimizer=dict(
        type=optim_type, lr=lr, betas=betas, weight_decay=weight_decay),
    clip_grad=dict(max_norm=max_norm, error_if_nonfinite=False),
    accumulative_counts=accumulative_counts,
    loss_scale='dynamic',
    dtype='float16')

# learning policy
# More information: https://github.com/open-mmlab/mmengine/blob/main/docs/en/tutorials/param_scheduler.md  # noqa: E501
param_scheduler = [
    dict(
        type=LinearLR,
        start_factor=1e-5,
        by_epoch=True,
        begin=0,
        end=warmup_ratio * max_epochs,
        convert_to_iter_based=True),
    dict(
        type=CosineAnnealingLR,
        eta_min=0.0,
        by_epoch=True,
        begin=warmup_ratio * max_epochs,
        T_max=max_epochs,
        convert_to_iter_based=True)
]

# train, val, test setting
train_cfg = dict(by_epoch=True, max_epochs=max_epochs, val_interval=1)

#######################################################################
#                           PART 5  Runtime                           #
#######################################################################
# Log the dialogue periodically during the training process, optional
custom_hooks = [
    dict(type=DatasetInfoHook, tokenizer=tokenizer),
    dict(
        type=EvaluateChatHook,
        tokenizer=tokenizer,
        every_n_iters=evaluation_freq,
        evaluation_inputs=evaluation_inputs,
        system=SYSTEM,
        prompt_template=prompt_template)
]

# configure default hooks
default_hooks = dict(
    # record the time of every iteration.
    timer=dict(type=IterTimerHook),
    # print log every 100 iterations.
    logger=dict(type=LoggerHook, interval=10),
    # enable the parameter scheduler.
    param_scheduler=dict(type=ParamSchedulerHook),
    # save checkpoint per epoch.
    checkpoint=dict(type=CheckpointHook, interval=1),
    # set sampler seed in distributed evrionment.
    sampler_seed=dict(type=DistSamplerSeedHook),
)

# configure environment
env_cfg = dict(
    # whether to enable cudnn benchmark
    cudnn_benchmark=False,
    # set multi process parameters
    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0),
    # set distributed parameters
    dist_cfg=dict(backend='nccl'),
)

# set visualizer
visualizer = None

# set log level
log_level = 'INFO'

# load from which checkpoint
load_from = None

# whether to resume training from the loaded checkpoint
resume = False

# Defaults to use random seed and disable `deterministic`
randomness = dict(seed=None, deterministic=False)

报错如下：

Traceback (most recent call last):
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/xtuner/tools/train.py", line 299, in <module>
    main()
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/xtuner/tools/train.py", line 295, in main
    runner.train()
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/mmengine/runner/_flexible_runner.py", line 1182, in train
    self.strategy.prepare(
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/mmengine/_strategy/deepspeed.py", line 389, in prepare
    self.param_schedulers = self.build_param_scheduler(
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/mmengine/_strategy/base.py", line 658, in build_param_scheduler
    param_schedulers = self._build_param_scheduler(
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/mmengine/_strategy/base.py", line 563, in _build_param_scheduler
    PARAM_SCHEDULERS.build(
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/mmengine/registry/registry.py", line 570, in build
    return self.build_func(cfg, *args, **kwargs, registry=self)
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/mmengine/registry/build_functions.py", line 294, in build_scheduler_from_cfg
    return scheduler_cls.build_iter_from_epoch(  # type: ignore
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/mmengine/optim/scheduler/param_scheduler.py", line 787, in build_iter_from_epoch
    return cls(*args, begin=begin, end=end, by_epoch=by_epoch, **kwargs)
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/mmengine/optim/scheduler/lr_scheduler.py", line 20, in __init__
    super().__init__(optimizer, 'lr', *args, **kwargs)
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/mmengine/optim/scheduler/param_scheduler.py", line 759, in __init__
    super().__init__(
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/mmengine/optim/scheduler/param_scheduler.py", line 68, in __init__
    raise ValueError('end should be larger than begin, but got'
ValueError: end should be larger than begin, but got begin=0, end=0
[2024-01-29 17:32:32,013] torch.distributed.elastic.multiprocessing.api: [ERROR] failed (exitcode: 1) local_rank: 0 (pid: 47059) of binary: /home/tangshi/miniconda3/envs/xtuner/bin/python
Traceback (most recent call last):
  File "/home/tangshi/miniconda3/envs/xtuner/bin/torchrun", line 8, in <module>
    sys.exit(main())
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper
    return f(*args, **kwargs)
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/torch/distributed/run.py", line 806, in main
    run(args)
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/torch/distributed/run.py", line 797, in run
    elastic_launch(
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 134, in __call__
    return launch_agent(self._config, self._entrypoint, list(args))
  File "/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent
    raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
============================================================
/home/tangshi/miniconda3/envs/xtuner/lib/python3.10/site-packages/xtuner/tools/train.py FAILED
------------------------------------------------------------
Failures:
[1]:
  time      : 2024-01-29_17:32:32
  host      : tangshi
  rank      : 1 (local_rank: 1)
  exitcode  : 1 (pid: 47060)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
[2]:
  time      : 2024-01-29_17:32:32
  host      : tangshi
  rank      : 2 (local_rank: 2)
  exitcode  : 1 (pid: 47061)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
[3]:
  time      : 2024-01-29_17:32:32
  host      : tangshi
  rank      : 3 (local_rank: 3)
  exitcode  : 1 (pid: 47062)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
------------------------------------------------------------
Root Cause (first observed failure):
[0]:
  time      : 2024-01-29_17:32:32
  host      : tangshi
  rank      : 0 (local_rank: 0)
  exitcode  : 1 (pid: 47059)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
============================================================

[LZHgrla]

应该是数据集个数太少，或者map_fn映射错误。可以检查一下这两项。

[Brain2nd]

应该是数据集个数太少，或者map_fn映射错误。可以检查一下这两项。

qwen使用同样数据集可以，但是书生2 20B不行，都是alpaca_zh_e3

[LZHgrla]

模型修改不会导致该错误，建议仔细检查config，对齐设置。如果还是还是有问题建议张贴完整的 qwen config 和 internlm config

[ccall19]

我在尝试lora微调 interlm2-20b 时也遇到同样的问题，报错如下：

Traceback (most recent call last):
File "/mnt/c/DPan/repo/xtuner/xtuner/tools/train.py", line 360, in
main()
File "/mnt/c/DPan/repo/xtuner/xtuner/tools/train.py", line 356, in main
runner.train()
File "/home/lllsh/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/runner/runner.py", line 1738, in train
self.param_schedulers = self.build_param_scheduler( # type: ignore
File "/home/lllsh/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/runner/runner.py", line 1262, in build_param_scheduler
param_schedulers = self._build_param_scheduler(
File "/home/lllsh/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/runner/runner.py", line 1179, in _build_param_scheduler
PARAM_SCHEDULERS.build(
File "/home/lllsh/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/registry/registry.py", line 570, in build
return self.build_func(cfg, *args, **kwargs, registry=self)
File "/home/lllsh/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/registry/build_functions.py", line 294, in build_scheduler_from_cfg
return scheduler_cls.build_iter_from_epoch( # type: ignore
File "/home/lllsh/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/optim/scheduler/param_scheduler.py", line 787, in build_iter_from_epoch
return cls(*args, begin=begin, end=end, by_epoch=by_epoch, **kwargs)
File "/home/lllsh/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/optim/scheduler/lr_scheduler.py", line 20, in init
super().init(optimizer, 'lr', *args, **kwargs)
File "/home/lllsh/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/optim/scheduler/param_scheduler.py", line 759, in init
super().init(
File "/home/lllsh/anaconda3/envs/xtuner-env/lib/python3.10/site-packages/mmengine/optim/scheduler/param_scheduler.py", line 68, in init
raise ValueError('end should be larger than begin, but got'
ValueError: end should be larger than begin, but got begin=0, end=0

config如下：

**> # Copyright (c) OpenMMLab. All rights reserved.
"""Data format:
[
    {
        "conversation": [
            {
                "system": "",
                "input": "xxx",
                "output": "xxx"
            },
            {
                "input": "xxx",
                "output": "xxx"
            }
        ]
    },
...
]
Please refer to https://github.com/InternLM/xtuner/blob/main/docs/en/user_guides/dataset_format.md for details.
"""  # noqa: E501
import torch
from datasets import load_dataset
from mmengine.hooks import (CheckpointHook, DistSamplerSeedHook, IterTimerHook,
                            LoggerHook, ParamSchedulerHook)
from mmengine.optim import AmpOptimWrapper, CosineAnnealingLR
from peft import LoraConfig
from torch.optim import AdamW
from torch.utils.data import BatchSampler
from transformers import AutoModelForCausalLM, AutoTokenizer,BitsAndBytesConfig

from xtuner.dataset import process_hf_dataset
from xtuner.dataset.collate_fns import default_collate_fn
from xtuner.dataset.samplers import InternRepoSampler
from xtuner.engine import (DatasetInfoHook, EvaluateChatHook, ThroughputHook,
                           VarlenAttnArgsToMessageHubHook)
from xtuner.engine.runner import TrainLoop
from xtuner.model import SupervisedFinetune
from xtuner.utils import PROMPT_TEMPLATE

#######################################################################
#                          PART 1  Settings                           #
#######################################################################
# Model
pretrained_model_name_or_path = '/mnt/c/Dpan/repo/InternLM-main/internlm2-20b'
use_varlen_attn = True

# Data
data_files = '/mnt/c/Dpan/repo/xtuner/datafor20b/train20b.json'
max_length = 128
pack_to_max_length = True

# parallel
sequence_parallel_size = 1

# Scheduler & Optimizer
# batch size per device, set to 1 if `use_varlen_attn` = True
# To clarify, enlarging the batch size essentially enlarges the `max_length`.
# For example, doubling the max length is tantamount to doubling the batch size
batch_size = 1
accumulative_counts = 1  # 1bs * 1acc * 64gpu = 64 batchsize
accumulative_counts *= sequence_parallel_size
dataloader_num_workers = 4
max_epochs = 1
optim_type = AdamW
lr = 4e-5
betas = (0.9, 0.95)
weight_decay = 0.01
max_norm = 1  # grad clip
warm_up_ratio = 0.025

# Save
save_steps = 500
save_total_limit = 2  # Maximum checkpoints to keep (-1 means unlimited)

# Evaluate the generation performance during the training
evaluation_freq = 500
SYSTEM = ''
evaluation_inputs = [
    '财联社资讯获悉，中国科学院软件研究所可信智能系统团队在开源自动驾驶（Self-drivingSystem）平台建设方面取得了进展，构建了开放易用且自主可控的自动驾驶算法开发、测试、仿真与验证平台ISS。\n一、自动驾驶板块迎来密集催化\n近期自动驾驶板块从产业端和政策端迎来密集催化。产业端，特斯拉开始向员工推送FSDV12版本更新。FSDV12实现了人工智能端到端的驾驶控制。其计算机视觉到驾驶决策均由神经网络与人工智能控制，而非通过工程师硬核编码，可简单理解为驾驶辅助系统把图像数据输入到神经网络，网络能够直接输出车辆控制指令。FSDV12的测试结果将影响自动驾驶行业的技术发展方向。此外，特斯拉中国本周对媒体表示，FSD入华一事正在推进中。中信建投预计特斯拉FSD入华对于国内智能驾驶产业链也将产生积极影响，有助于提升国内智能驾驶产业链的整体水平和竞争力，推动产业的可持续发展。\n政策端，11月17日，工信部等四部委发布《关于开展智能网联汽车准入和上路通行试点工作的通知》，对L3/L4准入和上路试点项目的申报方式、测试要求、责任界定等内容进行规范。其中，明确高阶智驾事故责任归属，将是未来L3及以上智能驾驶落地的重要基础。\n\n二、上半年我国辅助自动驾驶乘用车市场渗透率升至42.4%\n据媒体报道，工业和信息化部装备工业一司副司长郭守刚在“2023世界智能网联汽车大会”新闻发布会上表示，2022年我国搭载辅助自动驾驶系统的智能网联乘用车新车销售约700万辆，市场渗透率达到34.9%。今年上半年，市场渗透率近一步提升，达到了42.4%。\n海通国际证券杨林分析指出，辅助驾驶的渗透率不断提升，汽车软硬件技术不断发展，L3的智能驾驶汽车量产在即。当前，四部委发布《关于开展智能网联汽车准入和上路通行试点工作的通知》，从政策上为高级别自动驾驶上路保驾护航，高级别自动驾驶上路进程有望加速，将带动自动驾驶软硬件供应商的业绩持续增长。同时，高级别自动驾驶对于道路基础设施的要求有所提升，相关基础设施提供商有望受益。\n三、相关上市公司：光庭信息、路畅科技、秋田微\n光庭信息在自动驾驶方面提供ADAS(高级辅助驾驶)量产解决方案、APA(自动泊车)量产解决方案以及智能驾驶地图构建解决方案等。\n路畅科技参股子公司研发的智能驾驶产品已在部分景区进行了小量试用。\n秋田微主要从事液晶显示及触控产品的研发、设计、生产和销售。公司研发项目“自动驾驶激光雷达液晶光阀”已完成项目研发，目前处于市场推广阶段。\n近期热门系列：\n11月23日《马斯克称Cybertruck正运往北美特斯拉门店，预订量已超200万，机构称这两类公司有望受益》\n11月21日《智能驾驶商业化提速，机构称特斯拉FSD落地中国或进入倒计时》\n10月10日《小米汽车微信公众号正式上线，新晋造车巨头正悄然进入紧张的生产调试冲刺阶段》\n10月9日《受益于AI与数据要素双主线，运营商对该细分需求持续增加》'
]

#######################################################################
#                      PART 2  Model & Tokenizer                      #
#######################################################################
tokenizer = dict(
    type=AutoTokenizer.from_pretrained,
    pretrained_model_name_or_path=pretrained_model_name_or_path,
    trust_remote_code=True,
    padding_side='right')

model = dict(
    type=SupervisedFinetune,
    use_varlen_attn=use_varlen_attn,
    llm=dict(
        type=AutoModelForCausalLM.from_pretrained,
        pretrained_model_name_or_path=pretrained_model_name_or_path,
        trust_remote_code=True,
        torch_dtype=torch.float16,
        quantization_config=dict(  # 量化配置
            type=BitsAndBytesConfig,
            load_in_4bit=True,
            load_in_8bit=False,
            llm_int8_threshold=6.0,
            llm_int8_has_fp16_weight=False,
            bnb_4bit_compute_dtype=torch.float16,
            bnb_4bit_use_double_quant=True,
            bnb_4bit_quant_type='nf4')
    ),
    lora=dict(
        type=LoraConfig,
        r=8,
        lora_alpha=16,
        lora_dropout=0.1,
        bias='none',
        task_type='CAUSAL_LM'))
#######################################################################
#                      PART 3  Dataset & Dataloader                   #
#######################################################################
train_dataset = dict(
    type=process_hf_dataset,
    use_varlen_attn=use_varlen_attn,
    dataset=dict(type=load_dataset, path='json', data_files=data_files),
    tokenizer=tokenizer,
    max_length=max_length,
    dataset_map_fn=None,
    template_map_fn=None,
    # template_map_fn=dict(
    #     type=template_map_fn_factory, template=prompt_template),
    remove_unused_columns=True,
    shuffle_before_pack=True,
    pack_to_max_length=pack_to_max_length)

train_dataloader = dict(
    batch_size=batch_size,
    num_workers=dataloader_num_workers,
    dataset=train_dataset,
    sampler=dict(type=InternRepoSampler, shuffle=True, seed=1024),
    batch_sampler=dict(
        type=BatchSampler, drop_last=True, batch_size=batch_size),
    collate_fn=dict(type=default_collate_fn, use_varlen_attn=use_varlen_attn))

#######################################################################
#                    PART 4  Scheduler & Optimizer                    #
#######################################################################
# optimizer
optim_wrapper = dict(
    type=AmpOptimWrapper,
    optimizer=dict(
        type=optim_type, lr=lr, betas=betas, weight_decay=weight_decay),
    clip_grad=dict(max_norm=max_norm, error_if_nonfinite=False),
    accumulative_counts=accumulative_counts,
    loss_scale='dynamic',
)

# learning policy
# More information: https://github.com/open-mmlab/mmengine/blob/main/docs/en/tutorials/param_scheduler.md  # noqa: E501
param_scheduler = [
    dict(
        type='LinearLR',
        start_factor=1 / 40,
        by_epoch=True,
        begin=0,
        end=warm_up_ratio * max_epochs,
        convert_to_iter_based=True),
    dict(
        type=CosineAnnealingLR,
        eta_min=lr * 0.15,
        by_epoch=True,
        begin=warm_up_ratio * max_epochs,
        end=max_epochs,
        convert_to_iter_based=True)
]

# train, val, test setting
train_cfg = dict(type=TrainLoop, max_epochs=max_epochs)

#######################################################################
#                           PART 5  Runtime                           #
#######################################################################
# Log the dialogue periodically during the training process, optional
custom_hooks = [
    dict(type=DatasetInfoHook, tokenizer=tokenizer),
    # dict(
    #     type=DatasetInfoHook, tokenizer=tokenizer,
    #     is_intern_repo_dataset=True),
    # # dict(
    # #     type=EvaluateChatHook,
    # #     tokenizer=tokenizer,
    # #     every_n_iters=evaluation_freq,
    # #     evaluation_inputs=evaluation_inputs,
    # #     system=SYSTEM,
    # #     prompt_template=prompt_template),
    # dict(type=ThroughputHook)
]

if use_varlen_attn:
    custom_hooks += [dict(type=VarlenAttnArgsToMessageHubHook)]

# configure default hooks
default_hooks = dict(
    # record the time of every iteration.
    timer=dict(type=IterTimerHook),
    # print log every 100 iterations.
    logger=dict(type=LoggerHook, log_metric_by_epoch=False, interval=1),
    # enable the parameter scheduler.
    param_scheduler=dict(type=ParamSchedulerHook),
    # save checkpoint per `save_steps`.
    checkpoint=dict(
        type=CheckpointHook,
        by_epoch=False,
        interval=save_steps,
        max_keep_ckpts=save_total_limit),
    # set sampler seed in distributed evrionment.
    sampler_seed=dict(type=DistSamplerSeedHook),
)

# configure environment
env_cfg = dict(
    # whether to enable cudnn benchmark
    cudnn_benchmark=False,
    # set multi process parameters
    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0),
    # set distributed parameters
    dist_cfg=dict(backend='nccl'),
)

# set visualizer
visualizer = None

# set log level
log_level = 'INFO'

# load from which checkpoint
load_from = None

# whether to resume training from the loaded checkpoint
resume = False

# Defaults to use random seed and disable `deterministic`
randomness = dict(seed=None, deterministic=False)

log_processor = dict(
    by_epoch=False,
    window_size=1,
    mean_pattern=r'.*(loss|time|data_time|grad_norm|tflops).*')**